1 /** 2 Management of packages on the local computer. 3 4 Copyright: © 2012-2016 rejectedsoftware e.K. 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Sönke Ludwig, Matthias Dondorff 7 */ 8 module dub.packagemanager; 9 10 import dub.dependency; 11 import dub.internal.utils; 12 import dub.internal.vibecompat.core.file; 13 import dub.internal.vibecompat.core.log; 14 import dub.internal.vibecompat.data.json; 15 import dub.internal.vibecompat.inet.path; 16 import dub.package_; 17 18 import std.algorithm : countUntil, filter, sort, canFind, remove; 19 import std.array; 20 import std.conv; 21 import std.digest.sha; 22 import std.encoding : sanitize; 23 import std.exception; 24 import std.file; 25 import std..string; 26 import std.zip; 27 28 29 /// The PackageManager can retrieve present packages and get / remove 30 /// packages. 31 class PackageManager { 32 private { 33 Repository[] m_repositories; 34 NativePath[] m_searchPath; 35 Package[] m_packages; 36 Package[] m_temporaryPackages; 37 bool m_disableDefaultSearchPaths = false; 38 } 39 40 /** 41 Instantiate an instance with a single search path 42 43 This constructor is used when dub is invoked with the '--bar' CLI switch. 44 The instance will not look up the default repositories 45 (e.g. ~/.dub/packages), using only `path` instead. 46 47 Params: 48 path = Path of the single repository 49 */ 50 this(NativePath path) 51 { 52 this.m_searchPath = [ path ]; 53 this.m_disableDefaultSearchPaths = true; 54 this.refresh(true); 55 } 56 57 deprecated("Use the overload which accepts 3 `NativePath` arguments") 58 this(NativePath user_path, NativePath system_path, bool refresh_packages = true) 59 { 60 m_repositories = [ 61 Repository(user_path ~ "packages/"), 62 Repository(system_path ~ "packages/")]; 63 64 if (refresh_packages) refresh(true); 65 } 66 67 this(NativePath package_path, NativePath user_path, NativePath system_path, bool refresh_packages = true) 68 { 69 m_repositories = [ 70 Repository(package_path ~ ".dub/packages/"), 71 Repository(user_path ~ "packages/"), 72 Repository(system_path ~ "packages/")]; 73 74 if (refresh_packages) refresh(true); 75 } 76 77 /** Gets/sets the list of paths to search for local packages. 78 */ 79 @property void searchPath(NativePath[] paths) 80 { 81 if (paths == m_searchPath) return; 82 m_searchPath = paths.dup; 83 refresh(false); 84 } 85 /// ditto 86 @property const(NativePath)[] searchPath() const { return m_searchPath; } 87 88 /** Disables searching DUB's predefined search paths. 89 */ 90 deprecated("Instantiate a PackageManager instance with the single-argument constructor: `new PackageManager(path)`") 91 @property void disableDefaultSearchPaths(bool val) 92 { 93 this._disableDefaultSearchPaths(val); 94 } 95 96 // Non deprecated instance of the previous symbol, 97 // as `Dub.updatePackageSearchPath` calls it and while nothing in Dub app 98 // itself relies on it, just removing the call from `updatePackageSearchPath` 99 // could break the library use case. 100 package(dub) void _disableDefaultSearchPaths(bool val) 101 { 102 if (val == m_disableDefaultSearchPaths) return; 103 m_disableDefaultSearchPaths = val; 104 refresh(true); 105 } 106 107 /** Returns the effective list of search paths, including default ones. 108 */ 109 @property const(NativePath)[] completeSearchPath() 110 const { 111 auto ret = appender!(NativePath[])(); 112 ret.put(cast(NativePath[])m_searchPath); // work around Phobos 17251 113 if (!m_disableDefaultSearchPaths) { 114 foreach (ref repo; m_repositories) { 115 ret.put(cast(NativePath[])repo.searchPath); 116 ret.put(cast(NativePath)repo.packagePath); 117 } 118 } 119 return ret.data; 120 } 121 122 /** Sets additional (read-only) package cache paths to search for packages. 123 124 Cache paths have the same structure as the default cache paths, such as 125 ".dub/packages/". 126 127 Note that previously set custom paths will be removed when setting this 128 property. 129 */ 130 @property void customCachePaths(NativePath[] custom_cache_paths) 131 { 132 import std.algorithm.iteration : map; 133 import std.array : array; 134 135 m_repositories.length = LocalPackageType.max+1; 136 m_repositories ~= custom_cache_paths.map!(p => Repository(p)).array; 137 138 refresh(false); 139 } 140 141 142 /** Looks up a specific package. 143 144 Looks up a package matching the given version/path in the set of 145 registered packages. The lookup order is done according the the 146 usual rules (see getPackageIterator). 147 148 Params: 149 name = The name of the package 150 ver = The exact version of the package to query 151 path = An exact path that the package must reside in. Note that 152 the package must still be registered in the package manager. 153 enable_overrides = Apply the local package override list before 154 returning a package (enabled by default) 155 156 Returns: 157 The matching package or null if no match was found. 158 */ 159 Package getPackage(string name, Version ver, bool enable_overrides = true) 160 { 161 if (enable_overrides) { 162 foreach (ref repo; m_repositories) 163 foreach (ovr; repo.overrides) 164 if (ovr.package_ == name && ovr.version_.matches(ver)) { 165 Package pack; 166 if (!ovr.targetPath.empty) pack = getOrLoadPackage(ovr.targetPath); 167 else pack = getPackage(name, ovr.targetVersion, false); 168 if (pack) return pack; 169 170 logWarn("Package override %s %s -> %s %s doesn't reference an existing package.", 171 ovr.package_, ovr.version_, ovr.targetVersion, ovr.targetPath); 172 } 173 } 174 175 foreach (p; getPackageIterator(name)) 176 if (p.version_ == ver) 177 return p; 178 179 return null; 180 } 181 182 /// ditto 183 Package getPackage(string name, string ver, bool enable_overrides = true) 184 { 185 return getPackage(name, Version(ver), enable_overrides); 186 } 187 188 /// ditto 189 Package getPackage(string name, Version ver, NativePath path) 190 { 191 foreach (p; getPackageIterator(name)) 192 if (p.version_ == ver && p.path.startsWith(path)) 193 return p; 194 return null; 195 } 196 197 /// ditto 198 Package getPackage(string name, string ver, NativePath path) 199 { 200 return getPackage(name, Version(ver), path); 201 } 202 203 /// ditto 204 Package getPackage(string name, NativePath path) 205 { 206 foreach( p; getPackageIterator(name) ) 207 if (p.path.startsWith(path)) 208 return p; 209 return null; 210 } 211 212 213 /** Looks up the first package matching the given name. 214 */ 215 Package getFirstPackage(string name) 216 { 217 foreach (ep; getPackageIterator(name)) 218 return ep; 219 return null; 220 } 221 222 /** Looks up the latest package matching the given name. 223 */ 224 Package getLatestPackage(string name) 225 { 226 Package pkg; 227 foreach (ep; getPackageIterator(name)) 228 if (pkg is null || pkg.version_ < ep.version_) 229 pkg = ep; 230 return pkg; 231 } 232 233 /** For a given package path, returns the corresponding package. 234 235 If the package is already loaded, a reference is returned. Otherwise 236 the package gets loaded and cached for the next call to this function. 237 238 Params: 239 path = NativePath to the root directory of the package 240 recipe_path = Optional path to the recipe file of the package 241 allow_sub_packages = Also return a sub package if it resides in the given folder 242 243 Returns: The packages loaded from the given path 244 Throws: Throws an exception if no package can be loaded 245 */ 246 Package getOrLoadPackage(NativePath path, NativePath recipe_path = NativePath.init, bool allow_sub_packages = false) 247 { 248 path.endsWithSlash = true; 249 foreach (p; getPackageIterator()) 250 if (p.path == path && (!p.parentPackage || (allow_sub_packages && p.parentPackage.path != p.path))) 251 return p; 252 auto pack = Package.load(path, recipe_path); 253 addPackages(m_temporaryPackages, pack); 254 return pack; 255 } 256 257 /** For a given SCM repository, returns the corresponding package. 258 259 An SCM repository is provided as its remote URL, the repository is cloned 260 and in the dependency speicfied commit is checked out. 261 262 If the target directory already exists, just returns the package 263 without cloning. 264 265 Params: 266 name = Package name 267 dependency = Dependency that contains the repository URL and a specific commit 268 269 Returns: 270 The package loaded from the given SCM repository or null if the 271 package couldn't be loaded. 272 */ 273 Package loadSCMPackage(string name, Dependency dependency) 274 in { assert(!dependency.repository.empty); } 275 body { 276 Package pack; 277 278 with (dependency.repository) final switch (kind) 279 { 280 case Kind.git: 281 pack = loadGitPackage(name, dependency.versionSpec, dependency.repository.remote); 282 } 283 if (pack !is null) { 284 addPackages(m_temporaryPackages, pack); 285 } 286 return pack; 287 } 288 289 private Package loadGitPackage(string name, string versionSpec, string remote) 290 { 291 import dub.internal.git : cloneRepository; 292 293 if (!versionSpec.startsWith("~") && !versionSpec.isGitHash) { 294 return null; 295 } 296 297 string gitReference = versionSpec.chompPrefix("~"); 298 const destination = m_repositories[LocalPackageType.user].packagePath ~ 299 NativePath(name ~ "-" ~ gitReference) ~ (name~"/"); 300 301 foreach (p; getPackageIterator(name)) { 302 if (p.path == destination) { 303 return p; 304 } 305 } 306 307 if (!cloneRepository(remote, gitReference, destination.toNativeString())) { 308 return null; 309 } 310 311 return Package.load(destination); 312 } 313 314 /** Searches for the latest version of a package matching the given dependency. 315 */ 316 Package getBestPackage(string name, Dependency version_spec, bool enable_overrides = true) 317 { 318 Package ret; 319 foreach (p; getPackageIterator(name)) 320 if (version_spec.matches(p.version_) && (!ret || p.version_ > ret.version_)) 321 ret = p; 322 323 if (enable_overrides && ret) { 324 if (auto ovr = getPackage(name, ret.version_)) 325 return ovr; 326 } 327 return ret; 328 } 329 330 /// ditto 331 Package getBestPackage(string name, string version_spec) 332 { 333 return getBestPackage(name, Dependency(version_spec)); 334 } 335 336 /** Gets the a specific sub package. 337 338 In contrast to `Package.getSubPackage`, this function supports path 339 based sub packages. 340 341 Params: 342 base_package = The package from which to get a sub package 343 sub_name = Name of the sub package (not prefixed with the base 344 package name) 345 silent_fail = If set to true, the function will return `null` if no 346 package is found. Otherwise will throw an exception. 347 348 */ 349 Package getSubPackage(Package base_package, string sub_name, bool silent_fail) 350 { 351 foreach (p; getPackageIterator(base_package.name~":"~sub_name)) 352 if (p.parentPackage is base_package) 353 return p; 354 enforce(silent_fail, "Sub package \""~base_package.name~":"~sub_name~"\" doesn't exist."); 355 return null; 356 } 357 358 359 /** Determines if a package is managed by DUB. 360 361 Managed packages can be upgraded and removed. 362 */ 363 bool isManagedPackage(Package pack) 364 const { 365 auto ppath = pack.basePackage.path; 366 return isManagedPath(ppath); 367 } 368 369 /** Determines if a specific path is within a DUB managed package folder. 370 371 By default, managed folders are "~/.dub/packages" and 372 "/var/lib/dub/packages". 373 */ 374 bool isManagedPath(NativePath path) 375 const { 376 foreach (rep; m_repositories) { 377 NativePath rpath = rep.packagePath; 378 if (path.startsWith(rpath)) 379 return true; 380 } 381 return false; 382 } 383 384 /** Enables iteration over all known local packages. 385 386 Returns: A delegate suitable for use with `foreach` is returned. 387 */ 388 int delegate(int delegate(ref Package)) getPackageIterator() 389 { 390 int iterator(int delegate(ref Package) del) 391 { 392 foreach (tp; m_temporaryPackages) 393 if (auto ret = del(tp)) return ret; 394 395 // first search local packages 396 foreach (ref repo; m_repositories) 397 foreach (p; repo.localPackages) 398 if (auto ret = del(p)) return ret; 399 400 // and then all packages gathered from the search path 401 foreach( p; m_packages ) 402 if( auto ret = del(p) ) 403 return ret; 404 return 0; 405 } 406 407 return &iterator; 408 } 409 410 /** Enables iteration over all known local packages with a certain name. 411 412 Returns: A delegate suitable for use with `foreach` is returned. 413 */ 414 int delegate(int delegate(ref Package)) getPackageIterator(string name) 415 { 416 int iterator(int delegate(ref Package) del) 417 { 418 foreach (p; getPackageIterator()) 419 if (p.name == name) 420 if (auto ret = del(p)) return ret; 421 return 0; 422 } 423 424 return &iterator; 425 } 426 427 428 /** Returns a list of all package overrides for the given scope. 429 */ 430 const(PackageOverride)[] getOverrides(LocalPackageType scope_) 431 const { 432 return m_repositories[scope_].overrides; 433 } 434 435 /** Adds a new override for the given package. 436 */ 437 void addOverride(LocalPackageType scope_, string package_, Dependency version_spec, Version target) 438 { 439 m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target); 440 writeLocalPackageOverridesFile(scope_); 441 } 442 /// ditto 443 void addOverride(LocalPackageType scope_, string package_, Dependency version_spec, NativePath target) 444 { 445 m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target); 446 writeLocalPackageOverridesFile(scope_); 447 } 448 449 /** Removes an existing package override. 450 */ 451 void removeOverride(LocalPackageType scope_, string package_, Dependency version_spec) 452 { 453 Repository* rep = &m_repositories[scope_]; 454 foreach (i, ovr; rep.overrides) { 455 if (ovr.package_ != package_ || ovr.version_ != version_spec) 456 continue; 457 rep.overrides = rep.overrides[0 .. i] ~ rep.overrides[i+1 .. $]; 458 writeLocalPackageOverridesFile(scope_); 459 return; 460 } 461 throw new Exception(format("No override exists for %s %s", package_, version_spec)); 462 } 463 464 /// Extracts the package supplied as a path to it's zip file to the 465 /// destination and sets a version field in the package description. 466 Package storeFetchedPackage(NativePath zip_file_path, Json package_info, NativePath destination) 467 { 468 import std.range : walkLength; 469 470 auto package_name = package_info["name"].get!string; 471 auto package_version = package_info["version"].get!string; 472 473 logDebug("Placing package '%s' version '%s' to location '%s' from file '%s'", 474 package_name, package_version, destination.toNativeString(), zip_file_path.toNativeString()); 475 476 if( existsFile(destination) ){ 477 throw new Exception(format("%s (%s) needs to be removed from '%s' prior placement.", package_name, package_version, destination)); 478 } 479 480 // open zip file 481 ZipArchive archive; 482 { 483 logDebug("Opening file %s", zip_file_path); 484 auto f = openFile(zip_file_path, FileMode.read); 485 scope(exit) f.close(); 486 archive = new ZipArchive(f.readAll()); 487 } 488 489 logDebug("Extracting from zip."); 490 491 // In a github zip, the actual contents are in a subfolder 492 alias PSegment = typeof(NativePath.init.head); 493 PSegment[] zip_prefix; 494 outer: foreach(ArchiveMember am; archive.directory) { 495 auto path = NativePath(am.name).bySegment.array; 496 foreach (fil; packageInfoFiles) 497 if (path.length == 2 && path[$-1].name == fil.filename) { 498 zip_prefix = path[0 .. $-1]; 499 break outer; 500 } 501 } 502 503 logDebug("zip root folder: %s", zip_prefix); 504 505 NativePath getCleanedPath(string fileName) { 506 auto path = NativePath(fileName); 507 if (zip_prefix.length && !path.bySegment.startsWith(zip_prefix)) return NativePath.init; 508 static if (is(typeof(path[0 .. 1]))) return path[zip_prefix.length .. $]; 509 else return NativePath(path.bySegment.array[zip_prefix.length .. $]); 510 } 511 512 static void setAttributes(string path, ArchiveMember am) 513 { 514 import std.datetime : DosFileTimeToSysTime; 515 516 auto mtime = DosFileTimeToSysTime(am.time); 517 setTimes(path, mtime, mtime); 518 if (auto attrs = am.fileAttributes) 519 std.file.setAttributes(path, attrs); 520 } 521 522 // extract & place 523 mkdirRecurse(destination.toNativeString()); 524 logDebug("Copying all files..."); 525 int countFiles = 0; 526 foreach(ArchiveMember a; archive.directory) { 527 auto cleanedPath = getCleanedPath(a.name); 528 if(cleanedPath.empty) continue; 529 auto dst_path = destination ~ cleanedPath; 530 531 logDebug("Creating %s", cleanedPath); 532 if( dst_path.endsWithSlash ){ 533 if( !existsDirectory(dst_path) ) 534 mkdirRecurse(dst_path.toNativeString()); 535 } else { 536 if( !existsDirectory(dst_path.parentPath) ) 537 mkdirRecurse(dst_path.parentPath.toNativeString()); 538 { 539 auto dstFile = openFile(dst_path, FileMode.createTrunc); 540 scope(exit) dstFile.close(); 541 dstFile.put(archive.expand(a)); 542 } 543 setAttributes(dst_path.toNativeString(), a); 544 ++countFiles; 545 } 546 } 547 logDebug("%s file(s) copied.", to!string(countFiles)); 548 549 // overwrite dub.json (this one includes a version field) 550 auto pack = Package.load(destination, NativePath.init, null, package_info["version"].get!string); 551 552 if (pack.recipePath.head != defaultPackageFilename) 553 // Storeinfo saved a default file, this could be different to the file from the zip. 554 removeFile(pack.recipePath); 555 pack.storeInfo(); 556 addPackages(m_packages, pack); 557 return pack; 558 } 559 560 /// Removes the given the package. 561 void remove(in Package pack) 562 { 563 logDebug("Remove %s, version %s, path '%s'", pack.name, pack.version_, pack.path); 564 enforce(!pack.path.empty, "Cannot remove package "~pack.name~" without a path."); 565 566 // remove package from repositories' list 567 bool found = false; 568 bool removeFrom(Package[] packs, in Package pack) { 569 auto packPos = countUntil!("a.path == b.path")(packs, pack); 570 if(packPos != -1) { 571 packs = .remove(packs, packPos); 572 return true; 573 } 574 return false; 575 } 576 foreach(repo; m_repositories) { 577 if(removeFrom(repo.localPackages, pack)) { 578 found = true; 579 break; 580 } 581 } 582 if(!found) 583 found = removeFrom(m_packages, pack); 584 enforce(found, "Cannot remove, package not found: '"~ pack.name ~"', path: " ~ to!string(pack.path)); 585 586 logDebug("About to delete root folder for package '%s'.", pack.path); 587 rmdirRecurse(pack.path.toNativeString()); 588 logInfo("Removed package: '"~pack.name~"'"); 589 } 590 591 /// Compatibility overload. Use the version without a `force_remove` argument instead. 592 deprecated("Use `remove(pack)` directly instead, the boolean has no effect") 593 void remove(in Package pack, bool force_remove) 594 { 595 remove(pack); 596 } 597 598 Package addLocalPackage(NativePath path, string verName, LocalPackageType type) 599 { 600 path.endsWithSlash = true; 601 auto pack = Package.load(path); 602 enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString()); 603 if (verName.length) 604 pack.version_ = Version(verName); 605 606 // don't double-add packages 607 Package[]* packs = &m_repositories[type].localPackages; 608 foreach (p; *packs) { 609 if (p.path == path) { 610 enforce(p.version_ == pack.version_, "Adding the same local package twice with differing versions is not allowed."); 611 logInfo("Package is already registered: %s (version: %s)", p.name, p.version_); 612 return p; 613 } 614 } 615 616 addPackages(*packs, pack); 617 618 writeLocalPackageList(type); 619 620 logInfo("Registered package: %s (version: %s)", pack.name, pack.version_); 621 return pack; 622 } 623 624 void removeLocalPackage(NativePath path, LocalPackageType type) 625 { 626 path.endsWithSlash = true; 627 628 Package[]* packs = &m_repositories[type].localPackages; 629 size_t[] to_remove; 630 foreach( i, entry; *packs ) 631 if( entry.path == path ) 632 to_remove ~= i; 633 enforce(to_remove.length > 0, "No "~type.to!string()~" package found at "~path.toNativeString()); 634 635 string[Version] removed; 636 foreach_reverse( i; to_remove ) { 637 removed[(*packs)[i].version_] = (*packs)[i].name; 638 *packs = (*packs)[0 .. i] ~ (*packs)[i+1 .. $]; 639 } 640 641 writeLocalPackageList(type); 642 643 foreach(ver, name; removed) 644 logInfo("Deregistered package: %s (version: %s)", name, ver); 645 } 646 647 /// For the given type add another path where packages will be looked up. 648 void addSearchPath(NativePath path, LocalPackageType type) 649 { 650 m_repositories[type].searchPath ~= path; 651 writeLocalPackageList(type); 652 } 653 654 /// Removes a search path from the given type. 655 void removeSearchPath(NativePath path, LocalPackageType type) 656 { 657 m_repositories[type].searchPath = m_repositories[type].searchPath.filter!(p => p != path)().array(); 658 writeLocalPackageList(type); 659 } 660 661 void refresh(bool refresh_existing_packages) 662 { 663 logDiagnostic("Refreshing local packages (refresh existing: %s)...", refresh_existing_packages); 664 665 // load locally defined packages 666 void scanLocalPackages(LocalPackageType type) 667 { 668 NativePath list_path = m_repositories[type].packagePath; 669 Package[] packs; 670 NativePath[] paths; 671 try { 672 auto local_package_file = list_path ~ LocalPackagesFilename; 673 logDiagnostic("Looking for local package map at %s", local_package_file.toNativeString()); 674 if( !existsFile(local_package_file) ) return; 675 logDiagnostic("Try to load local package map at %s", local_package_file.toNativeString()); 676 auto packlist = jsonFromFile(list_path ~ LocalPackagesFilename); 677 enforce(packlist.type == Json.Type.array, LocalPackagesFilename~" must contain an array."); 678 foreach( pentry; packlist ){ 679 try { 680 auto name = pentry["name"].get!string; 681 auto path = NativePath(pentry["path"].get!string); 682 if (name == "*") { 683 paths ~= path; 684 } else { 685 auto ver = Version(pentry["version"].get!string); 686 687 Package pp; 688 if (!refresh_existing_packages) { 689 foreach (p; m_repositories[type].localPackages) 690 if (p.path == path) { 691 pp = p; 692 break; 693 } 694 } 695 696 if (!pp) { 697 auto infoFile = Package.findPackageFile(path); 698 if (!infoFile.empty) pp = Package.load(path, infoFile); 699 else { 700 logWarn("Locally registered package %s %s was not found. Please run 'dub remove-local \"%s\"'.", 701 name, ver, path.toNativeString()); 702 auto info = Json.emptyObject; 703 info["name"] = name; 704 pp = new Package(info, path); 705 } 706 } 707 708 if (pp.name != name) 709 logWarn("Local package at %s has different name than %s (%s)", path.toNativeString(), name, pp.name); 710 pp.version_ = ver; 711 712 addPackages(packs, pp); 713 } 714 } catch( Exception e ){ 715 logWarn("Error adding local package: %s", e.msg); 716 } 717 } 718 } catch( Exception e ){ 719 logDiagnostic("Loading of local package list at %s failed: %s", list_path.toNativeString(), e.msg); 720 } 721 m_repositories[type].localPackages = packs; 722 m_repositories[type].searchPath = paths; 723 } 724 if (!m_disableDefaultSearchPaths) 725 { 726 scanLocalPackages(LocalPackageType.system); 727 scanLocalPackages(LocalPackageType.user); 728 scanLocalPackages(LocalPackageType.package_); 729 } 730 731 auto old_packages = m_packages; 732 733 // rescan the system and user package folder 734 void scanPackageFolder(NativePath path) 735 { 736 if( path.existsDirectory() ){ 737 logDebug("iterating dir %s", path.toNativeString()); 738 try foreach( pdir; iterateDirectory(path) ){ 739 logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name); 740 if (!pdir.isDirectory) continue; 741 742 auto pack_path = path ~ (pdir.name ~ "/"); 743 744 auto packageFile = Package.findPackageFile(pack_path); 745 746 if (isManagedPath(path) && packageFile.empty) { 747 // Search for a single directory within this directory which happen to be a prefix of pdir 748 // This is to support new folder structure installed over the ancient one. 749 foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/"))) 750 if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) {// eg: package vibe-d will be in "vibe-d-x.y.z/vibe-d" 751 pack_path ~= subdir.name ~ "/"; 752 packageFile = Package.findPackageFile(pack_path); 753 break; 754 } 755 } 756 757 if (packageFile.empty) continue; 758 Package p; 759 try { 760 if (!refresh_existing_packages) 761 foreach (pp; old_packages) 762 if (pp.path == pack_path) { 763 p = pp; 764 break; 765 } 766 if (!p) p = Package.load(pack_path, packageFile); 767 addPackages(m_packages, p); 768 } catch( Exception e ){ 769 logError("Failed to load package in %s: %s", pack_path, e.msg); 770 logDiagnostic("Full error: %s", e.toString().sanitize()); 771 } 772 } 773 catch(Exception e) logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString()); 774 } 775 } 776 777 m_packages = null; 778 foreach (p; this.completeSearchPath) 779 scanPackageFolder(p); 780 781 void loadOverrides(LocalPackageType type) 782 { 783 m_repositories[type].overrides = null; 784 auto ovrfilepath = m_repositories[type].packagePath ~ LocalOverridesFilename; 785 if (existsFile(ovrfilepath)) { 786 foreach (entry; jsonFromFile(ovrfilepath)) { 787 PackageOverride ovr; 788 ovr.package_ = entry["name"].get!string; 789 ovr.version_ = Dependency(entry["version"].get!string); 790 if (auto pv = "targetVersion" in entry) ovr.targetVersion = Version(pv.get!string); 791 if (auto pv = "targetPath" in entry) ovr.targetPath = NativePath(pv.get!string); 792 m_repositories[type].overrides ~= ovr; 793 } 794 } 795 } 796 if (!m_disableDefaultSearchPaths) 797 { 798 loadOverrides(LocalPackageType.package_); 799 loadOverrides(LocalPackageType.user); 800 loadOverrides(LocalPackageType.system); 801 } 802 } 803 804 alias Hash = ubyte[]; 805 /// Generates a hash value for a given package. 806 /// Some files or folders are ignored during the generation (like .dub and 807 /// .svn folders) 808 Hash hashPackage(Package pack) 809 { 810 string[] ignored_directories = [".git", ".dub", ".svn"]; 811 // something from .dub_ignore or what? 812 string[] ignored_files = []; 813 SHA1 sha1; 814 foreach(file; dirEntries(pack.path.toNativeString(), SpanMode.depth)) { 815 if(file.isDir && ignored_directories.canFind(NativePath(file.name).head.name)) 816 continue; 817 else if(ignored_files.canFind(NativePath(file.name).head.name)) 818 continue; 819 820 sha1.put(cast(ubyte[])NativePath(file.name).head.name); 821 if(file.isDir) { 822 logDebug("Hashed directory name %s", NativePath(file.name).head); 823 } 824 else { 825 sha1.put(openFile(NativePath(file.name)).readAll()); 826 logDebug("Hashed file contents from %s", NativePath(file.name).head); 827 } 828 } 829 auto hash = sha1.finish(); 830 logDebug("Project hash: %s", hash); 831 return hash[].dup; 832 } 833 834 private void writeLocalPackageList(LocalPackageType type) 835 { 836 Json[] newlist; 837 foreach (p; m_repositories[type].searchPath) { 838 auto entry = Json.emptyObject; 839 entry["name"] = "*"; 840 entry["path"] = p.toNativeString(); 841 newlist ~= entry; 842 } 843 844 foreach (p; m_repositories[type].localPackages) { 845 if (p.parentPackage) continue; // do not store sub packages 846 auto entry = Json.emptyObject; 847 entry["name"] = p.name; 848 entry["version"] = p.version_.toString(); 849 entry["path"] = p.path.toNativeString(); 850 newlist ~= entry; 851 } 852 853 NativePath path = m_repositories[type].packagePath; 854 if( !existsDirectory(path) ) mkdirRecurse(path.toNativeString()); 855 writeJsonFile(path ~ LocalPackagesFilename, Json(newlist)); 856 } 857 858 private void writeLocalPackageOverridesFile(LocalPackageType type) 859 { 860 Json[] newlist; 861 foreach (ovr; m_repositories[type].overrides) { 862 auto jovr = Json.emptyObject; 863 jovr["name"] = ovr.package_; 864 jovr["version"] = ovr.version_.versionSpec; 865 if (!ovr.targetPath.empty) jovr["targetPath"] = ovr.targetPath.toNativeString(); 866 else jovr["targetVersion"] = ovr.targetVersion.toString(); 867 newlist ~= jovr; 868 } 869 auto path = m_repositories[type].packagePath; 870 if (!existsDirectory(path)) mkdirRecurse(path.toNativeString()); 871 writeJsonFile(path ~ LocalOverridesFilename, Json(newlist)); 872 } 873 874 /// Adds the package and scans for subpackages. 875 private void addPackages(ref Package[] dst_repos, Package pack) 876 const { 877 // Add the main package. 878 dst_repos ~= pack; 879 880 // Additionally to the internally defined subpackages, whose metadata 881 // is loaded with the main dub.json, load all externally defined 882 // packages after the package is available with all the data. 883 foreach (spr; pack.subPackages) { 884 Package sp; 885 886 if (spr.path.length) { 887 auto p = NativePath(spr.path); 888 p.normalize(); 889 enforce(!p.absolute, "Sub package paths must be sub paths of the parent package."); 890 auto path = pack.path ~ p; 891 if (!existsFile(path)) { 892 logError("Package %s declared a sub-package, definition file is missing: %s", pack.name, path.toNativeString()); 893 continue; 894 } 895 sp = Package.load(path, NativePath.init, pack); 896 } else sp = new Package(spr.recipe, pack.path, pack); 897 898 // Add the subpackage. 899 try { 900 dst_repos ~= sp; 901 } catch (Exception e) { 902 logError("Package '%s': Failed to load sub-package %s: %s", pack.name, 903 spr.path.length ? spr.path : spr.recipe.name, e.msg); 904 logDiagnostic("Full error: %s", e.toString().sanitize()); 905 } 906 } 907 } 908 } 909 910 struct PackageOverride { 911 string package_; 912 Dependency version_; 913 Version targetVersion; 914 NativePath targetPath; 915 916 this(string package_, Dependency version_, Version target_version) 917 { 918 this.package_ = package_; 919 this.version_ = version_; 920 this.targetVersion = target_version; 921 } 922 923 this(string package_, Dependency version_, NativePath target_path) 924 { 925 this.package_ = package_; 926 this.version_ = version_; 927 this.targetPath = target_path; 928 } 929 } 930 931 enum LocalPackageType { 932 package_, 933 user, 934 system 935 } 936 937 private enum LocalPackagesFilename = "local-packages.json"; 938 private enum LocalOverridesFilename = "local-overrides.json"; 939 940 941 private struct Repository { 942 NativePath packagePath; 943 NativePath[] searchPath; 944 Package[] localPackages; 945 PackageOverride[] overrides; 946 947 this(NativePath path) 948 { 949 this.packagePath = path; 950 } 951 }