1 /** 2 Management of packages on the local computer. 3 4 Copyright: © 2012-2013 rejectedsoftware e.K. 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Sönke Ludwig, Matthias Dondorff 7 */ 8 module dub.packagemanager; 9 10 import dub.dependency; 11 import dub.internal.utils; 12 import dub.internal.vibecompat.core.file; 13 import dub.internal.vibecompat.core.log; 14 import dub.internal.vibecompat.data.json; 15 import dub.internal.vibecompat.inet.path; 16 import dub.package_; 17 18 import std.algorithm : countUntil, filter, sort, canFind; 19 import std.array; 20 import std.conv; 21 import std.digest.sha; 22 import std.encoding : sanitize; 23 import std.exception; 24 import std.file; 25 import std.string; 26 import std.zip; 27 28 29 enum JournalJsonFilename = "journal.json"; 30 enum LocalPackagesFilename = "local-packages.json"; 31 32 33 private struct Repository { 34 Path path; 35 Path packagePath; 36 Path[] searchPath; 37 Package[] localPackages; 38 39 this(Path path) 40 { 41 this.path = path; 42 this.packagePath = path ~"packages/"; 43 } 44 } 45 46 enum LocalPackageType { 47 user, 48 system 49 } 50 51 /// The PackageManager can retrieve present packages and get / remove 52 /// packages. 53 class PackageManager { 54 private { 55 Repository[LocalPackageType] m_repositories; 56 Path[] m_searchPath; 57 Package[] m_packages; 58 Package[] m_temporaryPackages; 59 } 60 61 this(Path user_path, Path system_path) 62 { 63 m_repositories[LocalPackageType.user] = Repository(user_path); 64 m_repositories[LocalPackageType.system] = Repository(system_path); 65 refresh(true); 66 } 67 68 @property void searchPath(Path[] paths) { m_searchPath = paths.dup; refresh(false); } 69 @property const(Path)[] searchPath() const { return m_searchPath; } 70 71 @property const(Path)[] completeSearchPath() 72 const { 73 auto ret = appender!(Path[])(); 74 ret.put(m_searchPath); 75 ret.put(m_repositories[LocalPackageType.user].searchPath); 76 ret.put(m_repositories[LocalPackageType.user].packagePath); 77 ret.put(m_repositories[LocalPackageType.system].searchPath); 78 ret.put(m_repositories[LocalPackageType.system].packagePath); 79 return ret.data; 80 } 81 82 Package getPackage(string name, Version ver) 83 { 84 foreach( p; getPackageIterator(name) ) 85 if( p.ver == ver ) 86 return p; 87 return null; 88 } 89 90 Package getPackage(string name, string ver, Path in_path) 91 { 92 return getPackage(name, Version(ver), in_path); 93 } 94 Package getPackage(string name, Version ver, Path in_path) 95 { 96 foreach( p; getPackageIterator(name) ) 97 if (p.ver == ver && p.path.startsWith(in_path)) 98 return p; 99 return null; 100 } 101 102 Package getPackage(string name, string ver) 103 { 104 foreach (ep; getPackageIterator(name)) { 105 if (ep.vers == ver) 106 return ep; 107 } 108 return null; 109 } 110 111 Package getFirstPackage(string name) 112 { 113 foreach (ep; getPackageIterator(name)) 114 return ep; 115 return null; 116 } 117 118 Package getPackage(Path path) 119 { 120 foreach (p; getPackageIterator()) 121 if (!p.parentPackage && p.path == path) 122 return p; 123 auto pack = new Package(path); 124 addPackages(m_temporaryPackages, pack); 125 return pack; 126 } 127 128 Package getBestPackage(string name, string version_spec) 129 { 130 return getBestPackage(name, Dependency(version_spec)); 131 } 132 133 Package getBestPackage(string name, Dependency version_spec) 134 { 135 Package ret; 136 foreach( p; getPackageIterator(name) ) 137 if( version_spec.matches(p.ver) && (!ret || p.ver > ret.ver) ) 138 ret = p; 139 return ret; 140 } 141 142 /** Determines if a package is managed by DUB. 143 144 Managed packages can be upgraded and removed. 145 */ 146 bool isManagedPackage(Package pack) 147 const { 148 auto ppath = pack.basePackage.path; 149 foreach (rep; m_repositories) { 150 auto rpath = rep.packagePath; 151 if (ppath.startsWith(rpath)) 152 return true; 153 } 154 return false; 155 } 156 157 int delegate(int delegate(ref Package)) getPackageIterator() 158 { 159 int iterator(int delegate(ref Package) del) 160 { 161 int handlePackage(Package p) { 162 if (auto ret = del(p)) return ret; 163 foreach (sp; p.subPackages) 164 if (auto ret = del(sp)) 165 return ret; 166 return 0; 167 } 168 169 foreach (tp; m_temporaryPackages) 170 if (auto ret = handlePackage(tp)) return ret; 171 172 // first search local packages 173 foreach (tp; LocalPackageType.min .. LocalPackageType.max+1) 174 foreach (p; m_repositories[cast(LocalPackageType)tp].localPackages) 175 if (auto ret = handlePackage(p)) return ret; 176 177 // and then all packages gathered from the search path 178 foreach( p; m_packages ) 179 if( auto ret = handlePackage(p) ) 180 return ret; 181 return 0; 182 } 183 184 return &iterator; 185 } 186 187 int delegate(int delegate(ref Package)) getPackageIterator(string name) 188 { 189 int iterator(int delegate(ref Package) del) 190 { 191 foreach (p; getPackageIterator()) 192 if (p.name == name) 193 if (auto ret = del(p)) return ret; 194 return 0; 195 } 196 197 return &iterator; 198 } 199 200 /// Extracts the package supplied as a path to it's zip file to the 201 /// destination and sets a version field in the package description. 202 Package storeFetchedPackage(Path zip_file_path, Json package_info, Path destination) 203 { 204 auto package_name = package_info.name.get!string(); 205 auto package_version = package_info["version"].get!string(); 206 auto clean_package_version = package_version[package_version.startsWith("~") ? 1 : 0 .. $]; 207 208 logDiagnostic("Placing package '%s' version '%s' to location '%s' from file '%s'", 209 package_name, package_version, destination.toNativeString(), zip_file_path.toNativeString()); 210 211 if( existsFile(destination) ){ 212 throw new Exception(format("%s (%s) needs to be removed from '%s' prior placement.", package_name, package_version, destination)); 213 } 214 215 // open zip file 216 ZipArchive archive; 217 { 218 logDebug("Opening file %s", zip_file_path); 219 auto f = openFile(zip_file_path, FileMode.Read); 220 scope(exit) f.close(); 221 archive = new ZipArchive(f.readAll()); 222 } 223 224 logDebug("Extracting from zip."); 225 226 // In a github zip, the actual contents are in a subfolder 227 Path zip_prefix; 228 outer: foreach(ArchiveMember am; archive.directory) { 229 auto path = Path(am.name); 230 foreach (fil; packageInfoFilenames) 231 if (path.length == 2 && path.head.toString == fil) { 232 zip_prefix = path[0 .. $-1]; 233 break outer; 234 } 235 } 236 237 logDebug("zip root folder: %s", zip_prefix); 238 239 Path getCleanedPath(string fileName) { 240 auto path = Path(fileName); 241 if(zip_prefix != Path() && !path.startsWith(zip_prefix)) return Path(); 242 return path[zip_prefix.length..path.length]; 243 } 244 245 // extract & place 246 mkdirRecurse(destination.toNativeString()); 247 auto journal = new Journal; 248 logDiagnostic("Copying all files..."); 249 int countFiles = 0; 250 foreach(ArchiveMember a; archive.directory) { 251 auto cleanedPath = getCleanedPath(a.name); 252 if(cleanedPath.empty) continue; 253 auto dst_path = destination~cleanedPath; 254 255 logDebug("Creating %s", cleanedPath); 256 if( dst_path.endsWithSlash ){ 257 if( !existsDirectory(dst_path) ) 258 mkdirRecurse(dst_path.toNativeString()); 259 journal.add(Journal.Entry(Journal.Type.Directory, cleanedPath)); 260 } else { 261 if( !existsDirectory(dst_path.parentPath) ) 262 mkdirRecurse(dst_path.parentPath.toNativeString()); 263 auto dstFile = openFile(dst_path, FileMode.CreateTrunc); 264 scope(exit) dstFile.close(); 265 dstFile.put(archive.expand(a)); 266 journal.add(Journal.Entry(Journal.Type.RegularFile, cleanedPath)); 267 ++countFiles; 268 } 269 } 270 logDiagnostic("%s file(s) copied.", to!string(countFiles)); 271 272 // overwrite package.json (this one includes a version field) 273 auto pack = new Package(destination); 274 pack.info.version_ = package_info["version"].get!string; 275 276 if (pack.packageInfoFile.head != defaultPackageFilename()) { 277 // Storeinfo saved a default file, this could be different to the file from the zip. 278 removeFile(pack.packageInfoFile); 279 journal.remove(Journal.Entry(Journal.Type.RegularFile, Path(pack.packageInfoFile.head))); 280 journal.add(Journal.Entry(Journal.Type.RegularFile, Path(defaultPackageFilename()))); 281 } 282 pack.storeInfo(); 283 284 // Write journal 285 logDebug("Saving retrieval action journal..."); 286 journal.add(Journal.Entry(Journal.Type.RegularFile, Path(JournalJsonFilename))); 287 journal.save(destination ~ JournalJsonFilename); 288 289 addPackages(m_packages, pack); 290 291 return pack; 292 } 293 294 /// Removes the given the package. 295 void remove(in Package pack) 296 { 297 logDebug("Remove %s, version %s, path '%s'", pack.name, pack.vers, pack.path); 298 enforce(!pack.path.empty, "Cannot remove package "~pack.name~" without a path."); 299 300 // remove package from repositories' list 301 bool found = false; 302 bool removeFrom(Package[] packs, in Package pack) { 303 auto packPos = countUntil!("a.path == b.path")(packs, pack); 304 if(packPos != -1) { 305 packs = std.algorithm.remove(packs, packPos); 306 return true; 307 } 308 return false; 309 } 310 foreach(repo; m_repositories) { 311 if(removeFrom(repo.localPackages, pack)) { 312 found = true; 313 break; 314 } 315 } 316 if(!found) 317 found = removeFrom(m_packages, pack); 318 enforce(found, "Cannot remove, package not found: '"~ pack.name ~"', path: " ~ to!string(pack.path)); 319 320 // delete package files physically 321 logDebug("Looking up journal"); 322 auto journalFile = pack.path~JournalJsonFilename; 323 if (!existsFile(journalFile)) 324 throw new Exception("Removal failed, no retrieval journal found for '"~pack.name~"'. Please remove the folder '%s' manually.", pack.path.toNativeString()); 325 326 auto packagePath = pack.path; 327 auto journal = new Journal(journalFile); 328 logDebug("Erasing files"); 329 foreach( Journal.Entry e; filter!((Journal.Entry a) => a.type == Journal.Type.RegularFile)(journal.entries)) { 330 logDebug("Deleting file '%s'", e.relFilename); 331 auto absFile = pack.path~e.relFilename; 332 if(!existsFile(absFile)) { 333 logWarn("Previously retrieved file not found for removal: '%s'", absFile); 334 continue; 335 } 336 337 removeFile(absFile); 338 } 339 340 logDiagnostic("Erasing directories"); 341 Path[] allPaths; 342 foreach(Journal.Entry e; filter!((Journal.Entry a) => a.type == Journal.Type.Directory)(journal.entries)) 343 allPaths ~= pack.path~e.relFilename; 344 sort!("a.length>b.length")(allPaths); // sort to erase deepest paths first 345 foreach(Path p; allPaths) { 346 logDebug("Deleting folder '%s'", p); 347 if( !existsFile(p) || !isDir(p.toNativeString()) || !isEmptyDir(p) ) { 348 logError("Alien files found, directory is not empty or is not a directory: '%s'", p); 349 continue; 350 } 351 rmdir(p.toNativeString()); 352 } 353 354 // Erase .dub folder, this is completely erased. 355 auto dubDir = (pack.path ~ ".dub/").toNativeString(); 356 enforce(!existsFile(dubDir) || isDir(dubDir), ".dub should be a directory, but is a file."); 357 if(existsFile(dubDir) && isDir(dubDir)) { 358 logDebug(".dub directory found, removing directory including content."); 359 rmdirRecurse(dubDir); 360 } 361 362 logDebug("About to delete root folder for package '%s'.", pack.path); 363 if(!isEmptyDir(pack.path)) 364 throw new Exception("Alien files found in '"~pack.path.toNativeString()~"', needs to be deleted manually."); 365 366 rmdir(pack.path.toNativeString()); 367 logInfo("Removed package: '"~pack.name~"'"); 368 } 369 370 Package addLocalPackage(in Path path, string verName, LocalPackageType type) 371 { 372 auto pack = new Package(path); 373 enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString()); 374 if (verName.length) 375 pack.ver = Version(verName); 376 377 // don't double-add packages 378 Package[]* packs = &m_repositories[type].localPackages; 379 foreach (p; *packs) { 380 if (p.path == path) { 381 enforce(p.ver == pack.ver, "Adding the same local package twice with differing versions is not allowed."); 382 logInfo("Package is already registered: %s (version: %s)", p.name, p.ver); 383 return p; 384 } 385 } 386 387 addPackages(*packs, pack); 388 389 writeLocalPackageList(type); 390 391 logInfo("Registered package: %s (version: %s)", pack.name, pack.ver); 392 return pack; 393 } 394 395 void removeLocalPackage(in Path path, LocalPackageType type) 396 { 397 Package[]* packs = &m_repositories[type].localPackages; 398 size_t[] to_remove; 399 foreach( i, entry; *packs ) 400 if( entry.path == path ) 401 to_remove ~= i; 402 enforce(to_remove.length > 0, "No "~type.to!string()~" package found at "~path.toNativeString()); 403 404 string[Version] removed; 405 foreach_reverse( i; to_remove ) { 406 removed[(*packs)[i].ver] = (*packs)[i].name; 407 *packs = (*packs)[0 .. i] ~ (*packs)[i+1 .. $]; 408 } 409 410 writeLocalPackageList(type); 411 412 foreach(ver, name; removed) 413 logInfo("Unregistered package: %s (version: %s)", name, ver); 414 } 415 416 Package getTemporaryPackage(Path path, Version ver) 417 { 418 foreach (p; m_temporaryPackages) 419 if (p.path == path) { 420 enforce(p.ver == ver, format("Package in %s is refrenced with two conflicting versions: %s vs %s", path.toNativeString(), p.ver, ver)); 421 return p; 422 } 423 424 auto pack = new Package(path); 425 enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString()); 426 pack.ver = ver; 427 addPackages(m_temporaryPackages, pack); 428 return pack; 429 } 430 431 Package getTemporaryPackage(Path path) 432 { 433 foreach (p; m_temporaryPackages) 434 if (p.path == path) 435 return p; 436 437 auto pack = new Package(path); 438 enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString()); 439 addPackages(m_temporaryPackages, pack); 440 return pack; 441 } 442 443 /// For the given type add another path where packages will be looked up. 444 void addSearchPath(Path path, LocalPackageType type) 445 { 446 m_repositories[type].searchPath ~= path; 447 writeLocalPackageList(type); 448 } 449 450 /// Removes a search path from the given type. 451 void removeSearchPath(Path path, LocalPackageType type) 452 { 453 m_repositories[type].searchPath = m_repositories[type].searchPath.filter!(p => p != path)().array(); 454 writeLocalPackageList(type); 455 } 456 457 void refresh(bool refresh_existing_packages) 458 { 459 // load locally defined packages 460 void scanLocalPackages(LocalPackageType type) 461 { 462 Path list_path = m_repositories[type].packagePath; 463 Package[] packs; 464 Path[] paths; 465 try { 466 auto local_package_file = list_path ~ LocalPackagesFilename; 467 logDiagnostic("Looking for local package map at %s", local_package_file.toNativeString()); 468 if( !existsFile(local_package_file) ) return; 469 logDiagnostic("Try to load local package map at %s", local_package_file.toNativeString()); 470 auto packlist = jsonFromFile(list_path ~ LocalPackagesFilename); 471 enforce(packlist.type == Json.Type.array, LocalPackagesFilename~" must contain an array."); 472 foreach( pentry; packlist ){ 473 try { 474 auto name = pentry.name.get!string(); 475 auto path = Path(pentry.path.get!string()); 476 if (name == "*") { 477 paths ~= path; 478 } else { 479 auto ver = Version(pentry["version"].get!string()); 480 481 Package pp; 482 if (!refresh_existing_packages) { 483 foreach (p; m_repositories[type].localPackages) 484 if (p.path == path) { 485 pp = p; 486 break; 487 } 488 } 489 490 if (!pp) { 491 if (Package.isPackageAt(path)) pp = new Package(path); 492 else { 493 logWarn("Locally registered package %s %s was not found. Please run \"dub remove-local %s\".", 494 name, ver, path.toNativeString()); 495 auto info = Json.emptyObject; 496 info.name = name; 497 pp = new Package(info, path); 498 } 499 } 500 501 if (pp.name != name) 502 logWarn("Local package at %s has different name than %s (%s)", path.toNativeString(), name, pp.name); 503 pp.ver = ver; 504 505 addPackages(packs, pp); 506 } 507 } catch( Exception e ){ 508 logWarn("Error adding local package: %s", e.msg); 509 } 510 } 511 } catch( Exception e ){ 512 logDiagnostic("Loading of local package list at %s failed: %s", list_path.toNativeString(), e.msg); 513 } 514 m_repositories[type].localPackages = packs; 515 m_repositories[type].searchPath = paths; 516 } 517 scanLocalPackages(LocalPackageType.system); 518 scanLocalPackages(LocalPackageType.user); 519 520 auto old_packages = m_packages; 521 522 // rescan the system and user package folder 523 void scanPackageFolder(Path path) 524 { 525 if( path.existsDirectory() ){ 526 logDebug("iterating dir %s", path.toNativeString()); 527 try foreach( pdir; iterateDirectory(path) ){ 528 logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name); 529 if( !pdir.isDirectory ) continue; 530 auto pack_path = path ~ pdir.name; 531 if (!Package.isPackageAt(pack_path)) continue; 532 Package p; 533 try { 534 if (!refresh_existing_packages) 535 foreach (pp; old_packages) 536 if (pp.path == pack_path) { 537 p = pp; 538 break; 539 } 540 if (!p) p = new Package(pack_path); 541 addPackages(m_packages, p); 542 } catch( Exception e ){ 543 logError("Failed to load package in %s: %s", pack_path, e.msg); 544 logDiagnostic("Full error: %s", e.toString().sanitize()); 545 } 546 } 547 catch(Exception e) logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString()); 548 } 549 } 550 551 m_packages = null; 552 foreach (p; this.completeSearchPath) 553 scanPackageFolder(p); 554 } 555 556 alias ubyte[] Hash; 557 /// Generates a hash value for a given package. 558 /// Some files or folders are ignored during the generation (like .dub and 559 /// .svn folders) 560 Hash hashPackage(Package pack) 561 { 562 string[] ignored_directories = [".git", ".dub", ".svn"]; 563 // something from .dub_ignore or what? 564 string[] ignored_files = []; 565 SHA1 sha1; 566 foreach(file; dirEntries(pack.path.toNativeString(), SpanMode.depth)) { 567 if(file.isDir && ignored_directories.canFind(Path(file.name).head.toString())) 568 continue; 569 else if(ignored_files.canFind(Path(file.name).head.toString())) 570 continue; 571 572 sha1.put(cast(ubyte[])Path(file.name).head.toString()); 573 if(file.isDir) { 574 logDebug("Hashed directory name %s", Path(file.name).head); 575 } 576 else { 577 sha1.put(openFile(Path(file.name)).readAll()); 578 logDebug("Hashed file contents from %s", Path(file.name).head); 579 } 580 } 581 auto hash = sha1.finish(); 582 logDebug("Project hash: %s", hash); 583 return hash[0..$]; 584 } 585 586 private void writeLocalPackageList(LocalPackageType type) 587 { 588 Json[] newlist; 589 foreach (p; m_repositories[type].searchPath) { 590 auto entry = Json.emptyObject; 591 entry.name = "*"; 592 entry.path = p.toNativeString(); 593 newlist ~= entry; 594 } 595 596 foreach (p; m_repositories[type].localPackages) { 597 auto entry = Json.emptyObject; 598 entry["name"] = p.name; 599 entry["version"] = p.ver.toString(); 600 entry["path"] = p.path.toNativeString(); 601 newlist ~= entry; 602 } 603 604 Path path = m_repositories[type].packagePath; 605 if( !existsDirectory(path) ) mkdirRecurse(path.toNativeString()); 606 writeJsonFile(path ~ LocalPackagesFilename, Json(newlist)); 607 } 608 609 /// Adds the package and scans for subpackages. 610 private void addPackages(ref Package[] dst_repos, Package pack) const { 611 // Add the main package. 612 dst_repos ~= pack; 613 614 // Additionally to the internally defined subpackages, whose metadata 615 // is loaded with the main package.json, load all externally defined 616 // packages after the package is available with all the data. 617 foreach ( sub_path; pack.exportedPackages ) { 618 auto path = pack.path ~ sub_path; 619 if ( !existsFile(path) ) { 620 logError("Package %s declared a sub-package, definition file is missing: %s", pack.name, path.toNativeString()); 621 continue; 622 } 623 // Add the subpackage. 624 try { 625 dst_repos ~= new Package(path, pack); 626 } catch( Exception e ){ 627 logError("Package '%s': Failed to load sub-package in %s, error: %s", pack.name, path.toNativeString(), e.msg); 628 logDiagnostic("Full error: %s", e.toString().sanitize()); 629 } 630 } 631 } 632 } 633 634 635 /** 636 Retrieval journal for later removal, keeping track of placed files 637 files. 638 Example Json: 639 { 640 "version": 1, 641 "files": { 642 "file1": "typeoffile1", 643 ... 644 } 645 } 646 */ 647 private class Journal { 648 private enum Version = 1; 649 650 enum Type { 651 RegularFile, 652 Directory, 653 Alien 654 } 655 656 struct Entry { 657 this( Type t, Path f ) { type = t; relFilename = f; } 658 Type type; 659 Path relFilename; 660 } 661 662 @property const(Entry[]) entries() const { return m_entries; } 663 664 this() {} 665 666 /// Initializes a Journal from a json file. 667 this(Path journalFile) { 668 auto jsonJournal = jsonFromFile(journalFile); 669 enforce(cast(int)jsonJournal["Version"] == Version, "Mismatched version: "~to!string(cast(int)jsonJournal["Version"]) ~ "vs. " ~to!string(Version)); 670 foreach(string file, type; jsonJournal["Files"]) 671 m_entries ~= Entry(to!Type(cast(string)type), Path(file)); 672 } 673 674 void add(Entry e) { 675 foreach(Entry ent; entries) { 676 if( e.relFilename == ent.relFilename ) { 677 enforce(e.type == ent.type, "Duplicate('"~to!string(e.relFilename)~"'), different types: "~to!string(e.type)~" vs. "~to!string(ent.type)); 678 return; 679 } 680 } 681 m_entries ~= e; 682 } 683 684 void remove(Entry e) { 685 foreach(i, Entry ent; entries) { 686 if( e.relFilename == ent.relFilename ) { 687 m_entries = std.algorithm.remove(m_entries, i); 688 return; 689 } 690 } 691 enforce(false, "Cannot remove entry, not available: " ~ e.relFilename.toNativeString()); 692 } 693 694 /// Save the current state to the path. 695 void save(Path path) { 696 Json jsonJournal = serialize(); 697 auto fileJournal = openFile(path, FileMode.CreateTrunc); 698 scope(exit) fileJournal.close(); 699 fileJournal.writePrettyJsonString(jsonJournal); 700 } 701 702 private Json serialize() const { 703 Json[string] files; 704 foreach(Entry e; m_entries) 705 files[to!string(e.relFilename)] = to!string(e.type); 706 Json[string] json; 707 json["Version"] = Version; 708 json["Files"] = files; 709 return Json(json); 710 } 711 712 private { 713 Entry[] m_entries; 714 } 715 }