1 /** 2 A package manager. 3 4 Copyright: © 2012-2013 Matthias Dondorff 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Matthias Dondorff, Sönke Ludwig 7 */ 8 module dub.dub; 9 10 import dub.compilers.compiler; 11 import dub.dependency; 12 import dub.dependencyresolver; 13 import dub.internal.utils; 14 import dub.internal.vibecompat.core.file; 15 import dub.internal.vibecompat.core.log; 16 import dub.internal.vibecompat.data.json; 17 import dub.internal.vibecompat.inet.url; 18 import dub.package_; 19 import dub.packagemanager; 20 import dub.packagesupplier; 21 import dub.project; 22 import dub.generators.generator; 23 import dub.init; 24 25 26 // todo: cleanup imports. 27 import std.algorithm; 28 import std.array; 29 import std.conv; 30 import std.datetime; 31 import std.exception; 32 import std.file; 33 import std.process; 34 import std.string; 35 import std.typecons; 36 import std.zip; 37 import std.encoding : sanitize; 38 39 // Workaround for libcurl liker errors when building with LDC 40 version (LDC) pragma(lib, "curl"); 41 42 43 /// The default supplier for packages, which is the registry 44 /// hosted by code.dlang.org. 45 PackageSupplier[] defaultPackageSuppliers() 46 { 47 URL url = URL.parse("http://code.dlang.org/"); 48 logDiagnostic("Using dub registry url '%s'", url); 49 return [new RegistryPackageSupplier(url)]; 50 } 51 52 /// Option flags for fetch 53 enum FetchOptions 54 { 55 none = 0, 56 forceBranchUpgrade = 1<<0, 57 usePrerelease = 1<<1, 58 forceRemove = 1<<2, 59 printOnly = 1<<3, 60 } 61 62 /// The Dub class helps in getting the applications 63 /// dependencies up and running. An instance manages one application. 64 class Dub { 65 private { 66 bool m_dryRun = false; 67 PackageManager m_packageManager; 68 PackageSupplier[] m_packageSuppliers; 69 Path m_rootPath; 70 Path m_tempPath; 71 Path m_userDubPath, m_systemDubPath; 72 Json m_systemConfig, m_userConfig; 73 Path m_projectPath; 74 Project m_project; 75 Path m_overrideSearchPath; 76 } 77 78 /// Initiales the package manager for the vibe application 79 /// under root. 80 this(PackageSupplier[] additional_package_suppliers = null, string root_path = ".") 81 { 82 m_rootPath = Path(root_path); 83 if (!m_rootPath.absolute) m_rootPath = Path(getcwd()) ~ m_rootPath; 84 85 version(Windows){ 86 m_systemDubPath = Path(environment.get("ProgramData")) ~ "dub/"; 87 m_userDubPath = Path(environment.get("APPDATA")) ~ "dub/"; 88 m_tempPath = Path(environment.get("TEMP")); 89 } else version(Posix){ 90 m_systemDubPath = Path("/var/lib/dub/"); 91 m_userDubPath = Path(environment.get("HOME")) ~ ".dub/"; 92 if(!m_userDubPath.absolute) 93 m_userDubPath = Path(getcwd()) ~ m_userDubPath; 94 m_tempPath = Path("/tmp"); 95 } 96 97 m_userConfig = jsonFromFile(m_userDubPath ~ "settings.json", true); 98 m_systemConfig = jsonFromFile(m_systemDubPath ~ "settings.json", true); 99 100 PackageSupplier[] ps = additional_package_suppliers; 101 if (auto pp = "registryUrls" in m_userConfig) 102 ps ~= deserializeJson!(string[])(*pp) 103 .map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url))) 104 .array; 105 if (auto pp = "registryUrls" in m_systemConfig) 106 ps ~= deserializeJson!(string[])(*pp) 107 .map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url))) 108 .array; 109 ps ~= defaultPackageSuppliers(); 110 111 auto cacheDir = m_userDubPath ~ "cache/"; 112 foreach (p; ps) 113 p.cacheOp(cacheDir, CacheOp.load); 114 115 m_packageSuppliers = ps; 116 m_packageManager = new PackageManager(m_userDubPath, m_systemDubPath); 117 updatePackageSearchPath(); 118 } 119 120 /// Initializes DUB with only a single search path 121 this(Path override_path) 122 { 123 m_overrideSearchPath = override_path; 124 m_packageManager = new PackageManager(Path(), Path(), false); 125 updatePackageSearchPath(); 126 } 127 128 /// Perform cleanup and persist caches to disk 129 void shutdown() 130 { 131 auto cacheDir = m_userDubPath ~ "cache/"; 132 foreach (p; m_packageSuppliers) 133 p.cacheOp(cacheDir, CacheOp.store); 134 } 135 136 /// cleans all metadata caches 137 void cleanCaches() 138 { 139 auto cacheDir = m_userDubPath ~ "cache/"; 140 foreach (p; m_packageSuppliers) 141 p.cacheOp(cacheDir, CacheOp.clean); 142 } 143 144 @property void dryRun(bool v) { m_dryRun = v; } 145 146 /** Returns the root path (usually the current working directory). 147 */ 148 @property Path rootPath() const { return m_rootPath; } 149 /// ditto 150 @property void rootPath(Path root_path) 151 { 152 m_rootPath = root_path; 153 if (!m_rootPath.absolute) m_rootPath = Path(getcwd()) ~ m_rootPath; 154 } 155 156 /// Returns the name listed in the dub.json of the current 157 /// application. 158 @property string projectName() const { return m_project.name; } 159 160 @property Path projectPath() const { return m_projectPath; } 161 162 @property string[] configurations() const { return m_project.configurations; } 163 164 @property inout(PackageManager) packageManager() inout { return m_packageManager; } 165 166 @property inout(Project) project() inout { return m_project; } 167 168 /// Loads the package from the current working directory as the main 169 /// project package. 170 void loadPackageFromCwd() 171 { 172 loadPackage(m_rootPath); 173 } 174 175 /// Loads the package from the specified path as the main project package. 176 void loadPackage(Path path) 177 { 178 m_projectPath = path; 179 updatePackageSearchPath(); 180 m_project = new Project(m_packageManager, m_projectPath); 181 } 182 183 /// Loads a specific package as the main project package (can be a sub package) 184 void loadPackage(Package pack) 185 { 186 m_projectPath = pack.path; 187 updatePackageSearchPath(); 188 m_project = new Project(m_packageManager, pack); 189 } 190 191 void overrideSearchPath(Path path) 192 { 193 if (!path.absolute) path = Path(getcwd()) ~ path; 194 m_overrideSearchPath = path; 195 updatePackageSearchPath(); 196 } 197 198 string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); } 199 200 void upgrade(UpgradeOptions options) 201 { 202 // clear non-existent version selections 203 if (!(options & UpgradeOptions.upgrade)) { 204 next_pack: 205 foreach (p; m_project.selections.selectedPackages) { 206 auto dep = m_project.selections.getSelectedVersion(p); 207 if (!dep.path.empty) { 208 if (m_packageManager.getOrLoadPackage(dep.path)) continue; 209 } else { 210 if (m_packageManager.getPackage(p, dep.version_)) continue; 211 foreach (ps; m_packageSuppliers) { 212 try { 213 auto versions = ps.getVersions(p); 214 if (versions.canFind!(v => dep.matches(v))) 215 continue next_pack; 216 } catch (Exception e) { 217 logDiagnostic("Error querying versions for %s, %s: %s", p, ps.description, e.msg); 218 logDebug("Full error: %s", e.toString().sanitize()); 219 } 220 } 221 } 222 223 logWarn("Selected package %s %s doesn't exist. Using latest matching version instead.", p, dep); 224 m_project.selections.deselectVersion(p); 225 } 226 } 227 228 Dependency[string] versions; 229 if ((options & UpgradeOptions.useCachedResult) && m_project.isUpgradeCacheUpToDate()) { 230 logDiagnostic("Using cached upgrade results..."); 231 versions = m_project.getUpgradeCache(); 232 } else { 233 auto resolver = new DependencyVersionResolver(this, options); 234 versions = resolver.resolve(m_project.rootPackage, m_project.selections); 235 if (options & UpgradeOptions.useCachedResult) { 236 logDiagnostic("Caching upgrade results..."); 237 m_project.setUpgradeCache(versions); 238 } 239 } 240 241 if (options & UpgradeOptions.printUpgradesOnly) { 242 bool any = false; 243 string rootbasename = getBasePackageName(m_project.rootPackage.name); 244 245 foreach (p, ver; versions) { 246 if (!ver.path.empty) continue; 247 248 auto basename = getBasePackageName(p); 249 if (basename == rootbasename) continue; 250 251 if (!m_project.selections.hasSelectedVersion(basename)) { 252 logInfo("Package %s can be installed with version %s.", 253 basename, ver); 254 any = true; 255 continue; 256 } 257 auto sver = m_project.selections.getSelectedVersion(basename); 258 if (!sver.path.empty) continue; 259 if (ver.version_ <= sver.version_) continue; 260 logInfo("Package %s can be upgraded from %s to %s.", 261 basename, sver, ver); 262 any = true; 263 } 264 if (any) logInfo("Use \"dub upgrade\" to perform those changes."); 265 return; 266 } 267 268 foreach (p, ver; versions) { 269 assert(!p.canFind(":"), "Resolved packages contain a sub package!?: "~p); 270 Package pack; 271 if (!ver.path.empty) pack = m_packageManager.getOrLoadPackage(ver.path); 272 else { 273 pack = m_packageManager.getBestPackage(p, ver); 274 if (pack && m_packageManager.isManagedPackage(pack) 275 && ver.version_.isBranch && (options & UpgradeOptions.upgrade) != 0) 276 { 277 // TODO: only re-install if there is actually a new commit available 278 logInfo("Re-installing branch based dependency %s %s", p, ver.toString()); 279 m_packageManager.remove(pack, (options & UpgradeOptions.forceRemove) != 0); 280 pack = null; 281 } 282 } 283 284 FetchOptions fetchOpts; 285 fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none; 286 fetchOpts |= (options & UpgradeOptions.forceRemove) != 0 ? FetchOptions.forceRemove : FetchOptions.none; 287 if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version"); 288 if ((options & UpgradeOptions.select) && ver.path.empty && p != m_project.rootPackage.name) 289 m_project.selections.selectVersion(p, ver.version_); 290 } 291 292 m_project.reinit(); 293 294 if (options & UpgradeOptions.select) 295 m_project.saveSelections(); 296 } 297 298 /// Generate project files for a specified IDE. 299 /// Any existing project files will be overridden. 300 void generateProject(string ide, GeneratorSettings settings) { 301 auto generator = createProjectGenerator(ide, m_project); 302 if (m_dryRun) return; // TODO: pass m_dryRun to the generator 303 generator.generate(settings); 304 } 305 306 /// Executes tests on the current project. Throws an exception, if 307 /// unittests failed. 308 void testProject(GeneratorSettings settings, string config, Path custom_main_file) 309 { 310 if (custom_main_file.length && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file; 311 312 if (config.length == 0) { 313 // if a custom main file was given, favor the first library configuration, so that it can be applied 314 if (custom_main_file.length) config = m_project.getDefaultConfiguration(settings.platform, false); 315 // else look for a "unittest" configuration 316 if (!config.length && m_project.rootPackage.configurations.canFind("unittest")) config = "unittest"; 317 // if not found, fall back to the first "library" configuration 318 if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, false); 319 // if still nothing found, use the first executable configuration 320 if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, true); 321 } 322 323 auto generator = createProjectGenerator("build", m_project); 324 325 auto test_config = format("__test__%s__", config); 326 327 BuildSettings lbuildsettings = settings.buildSettings; 328 m_project.addBuildSettings(lbuildsettings, settings.platform, config, null, true); 329 if (lbuildsettings.targetType == TargetType.none) { 330 logInfo(`Configuration '%s' has target type "none". Skipping test.`, config); 331 return; 332 } 333 334 if (lbuildsettings.targetType == TargetType.executable) { 335 if (config == "unittest") logInfo("Running custom 'unittest' configuration.", config); 336 else logInfo(`Configuration '%s' does not output a library. Falling back to "dub -b unittest -c %s".`, config, config); 337 if (!custom_main_file.empty) logWarn("Ignoring custom main file."); 338 settings.config = config; 339 } else if (lbuildsettings.sourceFiles.empty) { 340 logInfo(`No source files found in configuration '%s'. Falling back to "dub -b unittest".`, config); 341 if (!custom_main_file.empty) logWarn("Ignoring custom main file."); 342 settings.config = m_project.getDefaultConfiguration(settings.platform); 343 } else { 344 logInfo(`Generating test runner configuration '%s' for '%s' (%s).`, test_config, config, lbuildsettings.targetType); 345 346 BuildSettingsTemplate tcinfo = m_project.rootPackage.info.getConfiguration(config).buildSettings; 347 tcinfo.targetType = TargetType.executable; 348 tcinfo.targetName = test_config; 349 tcinfo.versions[""] ~= "VibeCustomMain"; // HACK for vibe.d's legacy main() behavior 350 string custommodname; 351 if (custom_main_file.length) { 352 import std.path; 353 tcinfo.sourceFiles[""] ~= custom_main_file.relativeTo(m_project.rootPackage.path).toNativeString(); 354 tcinfo.importPaths[""] ~= custom_main_file.parentPath.toNativeString(); 355 custommodname = custom_main_file.head.toString().baseName(".d"); 356 } 357 358 string[] import_modules; 359 foreach (file; lbuildsettings.sourceFiles) { 360 if (file.endsWith(".d") && Path(file).head.toString() != "package.d") 361 import_modules ~= lbuildsettings.determineModuleName(Path(file), m_project.rootPackage.path); 362 } 363 364 // generate main file 365 Path mainfile = getTempFile("dub_test_root", ".d"); 366 tcinfo.sourceFiles[""] ~= mainfile.toNativeString(); 367 tcinfo.mainSourceFile = mainfile.toNativeString(); 368 if (!m_dryRun) { 369 auto fil = openFile(mainfile, FileMode.CreateTrunc); 370 scope(exit) fil.close(); 371 fil.write("module dub_test_root;\n"); 372 fil.write("import std.typetuple;\n"); 373 foreach (mod; import_modules) fil.write(format("static import %s;\n", mod)); 374 fil.write("alias allModules = TypeTuple!("); 375 foreach (i, mod; import_modules) { 376 if (i > 0) fil.write(", "); 377 fil.write(mod); 378 } 379 fil.write(");\n"); 380 if (custommodname.length) { 381 fil.write(format("import %s;\n", custommodname)); 382 } else { 383 fil.write(q{ 384 import std.stdio; 385 import core.runtime; 386 387 void main() { writeln("All unit tests have been run successfully."); } 388 shared static this() { 389 version (Have_tested) { 390 import tested; 391 import core.runtime; 392 import std.exception; 393 Runtime.moduleUnitTester = () => true; 394 //runUnitTests!app(new JsonTestResultWriter("results.json")); 395 enforce(runUnitTests!allModules(new ConsoleTestResultWriter), "Unit tests failed."); 396 } 397 } 398 }); 399 } 400 } 401 m_project.rootPackage.info.configurations ~= ConfigurationInfo(test_config, tcinfo); 402 m_project = new Project(m_packageManager, m_project.rootPackage); 403 404 settings.config = test_config; 405 } 406 407 generator.generate(settings); 408 } 409 410 /// Outputs a JSON description of the project, including its dependencies. 411 void describeProject(BuildPlatform platform, string config) 412 { 413 auto dst = Json.emptyObject; 414 dst.configuration = config; 415 dst.compiler = platform.compiler; 416 dst.architecture = platform.architecture.serializeToJson(); 417 dst.platform = platform.platform.serializeToJson(); 418 419 m_project.describe(dst, platform, config); 420 421 import std.stdio; 422 write(dst.toPrettyString()); 423 } 424 425 /// Cleans intermediate/cache files of the given package 426 void cleanPackage(Path path) 427 { 428 logInfo("Cleaning package at %s...", path.toNativeString()); 429 enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString()); 430 431 // TODO: clear target files and copy files 432 433 if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString()); 434 if (existsFile(path ~ ".dub/obj")) rmdirRecurse((path ~ ".dub/obj").toNativeString()); 435 } 436 437 438 /// Returns all cached packages as a "packageId" = "version" associative array 439 string[string] cachedPackages() const { return m_project.cachedPackagesIDs; } 440 441 /// Fetches the package matching the dependency and places it in the specified location. 442 Package fetch(string packageId, const Dependency dep, PlacementLocation location, FetchOptions options, string reason = "") 443 { 444 Json pinfo; 445 PackageSupplier supplier; 446 foreach(ps; m_packageSuppliers){ 447 try { 448 pinfo = ps.getPackageDescription(packageId, dep, (options & FetchOptions.usePrerelease) != 0); 449 supplier = ps; 450 break; 451 } catch(Exception e) { 452 logDiagnostic("Package %s not found for %s: %s", packageId, ps.description, e.msg); 453 logDebug("Full error: %s", e.toString().sanitize()); 454 } 455 } 456 enforce(pinfo.type != Json.Type.undefined, "No package "~packageId~" was found matching the dependency "~dep.toString()); 457 string ver = pinfo["version"].get!string; 458 459 Path placement; 460 final switch (location) { 461 case PlacementLocation.local: placement = m_rootPath; break; 462 case PlacementLocation.user: placement = m_userDubPath ~ "packages/"; break; 463 case PlacementLocation.system: placement = m_systemDubPath ~ "packages/"; break; 464 } 465 466 // always upgrade branch based versions - TODO: actually check if there is a new commit available 467 Package existing; 468 try existing = m_packageManager.getPackage(packageId, ver, placement); 469 catch (Exception e) { 470 logWarn("Failed to load existing package %s: %s", ver, e.msg); 471 logDiagnostic("Full error: %s", e.toString().sanitize); 472 } 473 474 if (options & FetchOptions.printOnly) { 475 if (existing && existing.vers != ver) 476 logInfo("A new version for %s is available (%s -> %s). Run \"dub upgrade %s\" to switch.", 477 packageId, existing.vers, ver, packageId); 478 return null; 479 } 480 481 if (existing) { 482 if (!ver.startsWith("~") || !(options & FetchOptions.forceBranchUpgrade) || location == PlacementLocation.local) { 483 // TODO: support git working trees by performing a "git pull" instead of this 484 logDiagnostic("Package %s %s (%s) is already present with the latest version, skipping upgrade.", 485 packageId, ver, placement); 486 return existing; 487 } else { 488 logInfo("Removing %s %s to prepare replacement with a new version.", packageId, ver); 489 if (!m_dryRun) m_packageManager.remove(existing, (options & FetchOptions.forceRemove) != 0); 490 } 491 } 492 493 if (reason.length) logInfo("Fetching %s %s (%s)...", packageId, ver, reason); 494 else logInfo("Fetching %s %s...", packageId, ver); 495 if (m_dryRun) return null; 496 497 logDiagnostic("Acquiring package zip file"); 498 auto dload = m_projectPath ~ ".dub/temp/downloads"; 499 auto tempfname = packageId ~ "-" ~ (ver.startsWith('~') ? ver[1 .. $] : ver) ~ ".zip"; 500 auto tempFile = m_tempPath ~ tempfname; 501 string sTempFile = tempFile.toNativeString(); 502 if (exists(sTempFile)) std.file.remove(sTempFile); 503 supplier.retrievePackage(tempFile, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail? 504 scope(exit) std.file.remove(sTempFile); 505 506 logInfo("Placing %s %s to %s...", packageId, ver, placement.toNativeString()); 507 auto clean_package_version = ver[ver.startsWith("~") ? 1 : 0 .. $]; 508 clean_package_version = clean_package_version.replace("+", "_"); // + has special meaning for Optlink 509 Path dstpath = placement ~ (packageId ~ "-" ~ clean_package_version); 510 511 return m_packageManager.storeFetchedPackage(tempFile, pinfo, dstpath); 512 } 513 514 /// Removes a given package from the list of present/cached modules. 515 /// @removeFromApplication: if true, this will also remove an entry in the 516 /// list of dependencies in the application's dub.json 517 void remove(in Package pack, bool force_remove) 518 { 519 logInfo("Removing %s in %s", pack.name, pack.path.toNativeString()); 520 if (!m_dryRun) m_packageManager.remove(pack, force_remove); 521 } 522 523 /// @see remove(string, string, RemoveLocation) 524 enum RemoveVersionWildcard = "*"; 525 526 /// This will remove a given package with a specified version from the 527 /// location. 528 /// It will remove at most one package, unless @param version_ is 529 /// specified as wildcard "*". 530 /// @param package_id Package to be removed 531 /// @param version_ Identifying a version or a wild card. An empty string 532 /// may be passed into. In this case the package will be removed from the 533 /// location, if there is only one version retrieved. This will throw an 534 /// exception, if there are multiple versions retrieved. 535 /// Note: as wildcard string only RemoveVersionWildcard ("*") is supported. 536 /// @param location_ 537 void remove(string package_id, string version_, PlacementLocation location_, bool force_remove) 538 { 539 enforce(!package_id.empty); 540 if (location_ == PlacementLocation.local) { 541 logInfo("To remove a locally placed package, make sure you don't have any data" 542 ~ "\nleft in it's directory and then simply remove the whole directory."); 543 throw new Exception("dub cannot remove locally installed packages."); 544 } 545 546 Package[] packages; 547 const bool wildcardOrEmpty = version_ == RemoveVersionWildcard || version_.empty; 548 549 // Retrieve packages to be removed. 550 foreach(pack; m_packageManager.getPackageIterator(package_id)) 551 if( wildcardOrEmpty || pack.vers == version_ ) 552 packages ~= pack; 553 554 // Check validity of packages to be removed. 555 if(packages.empty) { 556 throw new Exception("Cannot find package to remove. (" 557 ~ "id: '" ~ package_id ~ "', version: '" ~ version_ ~ "', location: '" ~ to!string(location_) ~ "'" 558 ~ ")"); 559 } 560 if(version_.empty && packages.length > 1) { 561 logError("Cannot remove package '" ~ package_id ~ "', there are multiple possibilities at location\n" 562 ~ "'" ~ to!string(location_) ~ "'."); 563 logError("Available versions:"); 564 foreach(pack; packages) 565 logError(" %s", pack.vers); 566 throw new Exception("Please specify a individual version using --version=... or use the" 567 ~ " wildcard --version=" ~ RemoveVersionWildcard ~ " to remove all versions."); 568 } 569 570 logDebug("Removing %s packages.", packages.length); 571 foreach(pack; packages) { 572 try { 573 remove(pack, force_remove); 574 logInfo("Removed %s, version %s.", package_id, pack.vers); 575 } catch (Exception e) { 576 logError("Failed to remove %s %s: %s", package_id, pack.vers, e.msg); 577 logInfo("Continuing with other packages (if any)."); 578 } 579 } 580 } 581 582 void addLocalPackage(string path, string ver, bool system) 583 { 584 if (m_dryRun) return; 585 m_packageManager.addLocalPackage(makeAbsolute(path), ver, system ? LocalPackageType.system : LocalPackageType.user); 586 } 587 588 void removeLocalPackage(string path, bool system) 589 { 590 if (m_dryRun) return; 591 m_packageManager.removeLocalPackage(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user); 592 } 593 594 void addSearchPath(string path, bool system) 595 { 596 if (m_dryRun) return; 597 m_packageManager.addSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user); 598 } 599 600 void removeSearchPath(string path, bool system) 601 { 602 if (m_dryRun) return; 603 m_packageManager.removeSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user); 604 } 605 606 void createEmptyPackage(Path path, string[] deps, string type) 607 { 608 if (!path.absolute) path = m_rootPath ~ path; 609 path.normalize(); 610 611 if (m_dryRun) return; 612 string[string] depVers; 613 string[] notFound; // keep track of any failed packages in here 614 foreach(ps; this.m_packageSuppliers){ 615 foreach(dep; deps){ 616 try{ 617 auto versionStrings = ps.getVersions(dep); 618 depVers[dep] = versionStrings[$-1].toString; 619 } catch(Exception e){ 620 notFound ~= dep; 621 } 622 } 623 } 624 if(notFound.length > 1){ 625 throw new Exception(format("Couldn't find packages: %-(%s, %).", notFound)); 626 } 627 else if(notFound.length == 1){ 628 throw new Exception(format("Couldn't find package: %-(%s, %).", notFound)); 629 } 630 631 initPackage(path, depVers, type); 632 633 //Act smug to the user. 634 logInfo("Successfully created an empty project in '%s'.", path.toNativeString()); 635 } 636 637 void runDdox(bool run) 638 { 639 if (m_dryRun) return; 640 641 auto ddox_pack = m_packageManager.getBestPackage("ddox", ">=0.0.0"); 642 if (!ddox_pack) ddox_pack = m_packageManager.getBestPackage("ddox", "~master"); 643 if (!ddox_pack) { 644 logInfo("DDOX is not present, getting it and storing user wide"); 645 ddox_pack = fetch("ddox", Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none); 646 } 647 648 version(Windows) auto ddox_exe = "ddox.exe"; 649 else auto ddox_exe = "ddox"; 650 651 if( !existsFile(ddox_pack.path~ddox_exe) ){ 652 logInfo("DDOX in %s is not built, performing build now.", ddox_pack.path.toNativeString()); 653 654 auto ddox_dub = new Dub(m_packageSuppliers); 655 ddox_dub.loadPackage(ddox_pack.path); 656 ddox_dub.upgrade(UpgradeOptions.select); 657 658 auto compiler_binary = "dmd"; 659 660 GeneratorSettings settings; 661 settings.config = "application"; 662 settings.compiler = getCompiler(compiler_binary); 663 settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary); 664 settings.buildType = "debug"; 665 ddox_dub.generateProject("build", settings); 666 667 //runCommands(["cd "~ddox_pack.path.toNativeString()~" && dub build -v"]); 668 } 669 670 auto p = ddox_pack.path; 671 p.endsWithSlash = true; 672 auto dub_path = p.toNativeString(); 673 674 string[] commands; 675 string[] filterargs = m_project.rootPackage.info.ddoxFilterArgs.dup; 676 if (filterargs.empty) filterargs = ["--min-protection=Protected", "--only-documented"]; 677 commands ~= dub_path~"ddox filter "~filterargs.join(" ")~" docs.json"; 678 if (!run) { 679 commands ~= dub_path~"ddox generate-html --navigation-type=ModuleTree docs.json docs"; 680 version(Windows) commands ~= "xcopy /S /D "~dub_path~"public\\* docs\\"; 681 else commands ~= "rsync -ru '"~dub_path~"public/' docs/"; 682 } 683 runCommands(commands); 684 685 if (run) { 686 auto proc = spawnProcess([dub_path~"ddox", "serve-html", "--navigation-type=ModuleTree", "docs.json", "--web-file-dir="~dub_path~"public"]); 687 browse("http://127.0.0.1:8080/"); 688 wait(proc); 689 } 690 } 691 692 private void updatePackageSearchPath() 693 { 694 if (m_overrideSearchPath.length) { 695 m_packageManager.disableDefaultSearchPaths = true; 696 m_packageManager.searchPath = [m_overrideSearchPath]; 697 } else { 698 auto p = environment.get("DUBPATH"); 699 Path[] paths; 700 701 version(Windows) enum pathsep = ";"; 702 else enum pathsep = ":"; 703 if (p.length) paths ~= p.split(pathsep).map!(p => Path(p))().array(); 704 m_packageManager.disableDefaultSearchPaths = false; 705 m_packageManager.searchPath = paths; 706 } 707 } 708 709 private Path makeAbsolute(Path p) const { return p.absolute ? p : m_rootPath ~ p; } 710 private Path makeAbsolute(string p) const { return makeAbsolute(Path(p)); } 711 } 712 713 string determineModuleName(BuildSettings settings, Path file, Path base_path) 714 { 715 assert(base_path.absolute); 716 if (!file.absolute) file = base_path ~ file; 717 718 size_t path_skip = 0; 719 foreach (ipath; settings.importPaths.map!(p => Path(p))) { 720 if (!ipath.absolute) ipath = base_path ~ ipath; 721 assert(!ipath.empty); 722 if (file.startsWith(ipath) && ipath.length > path_skip) 723 path_skip = ipath.length; 724 } 725 726 enforce(path_skip > 0, 727 format("Source file '%s' not found in any import path.", file.toNativeString())); 728 729 auto mpath = file[path_skip .. file.length]; 730 auto ret = appender!string; 731 732 //search for module keyword in file 733 string moduleName = getModuleNameFromFile(file.to!string); 734 735 if(moduleName.length) return moduleName; 736 737 //create module name from path 738 foreach (i; 0 .. mpath.length) { 739 import std.path; 740 auto p = mpath[i].toString(); 741 if (p == "package.d") break; 742 if (i > 0) ret ~= "."; 743 if (i+1 < mpath.length) ret ~= p; 744 else ret ~= p.baseName(".d"); 745 } 746 747 return ret.data; 748 } 749 750 /** 751 * Search for module keyword in D Code 752 */ 753 string getModuleNameFromContent(string content) { 754 import std.regex; 755 import std.string; 756 757 content = content.strip; 758 if (!content.length) return null; 759 760 static bool regex_initialized = false; 761 static Regex!char comments_pattern, module_pattern; 762 763 if (!regex_initialized) { 764 comments_pattern = regex(`(/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/)|(//.*)`, "g"); 765 module_pattern = regex(`module\s+([\w\.]+)\s*;`, "g"); 766 regex_initialized = true; 767 } 768 769 content = replaceAll(content, comments_pattern, ""); 770 auto result = matchFirst(content, module_pattern); 771 772 string moduleName; 773 if(!result.empty) moduleName = result.front; 774 775 if (moduleName.length >= 7) moduleName = moduleName[7..$-1]; 776 777 return moduleName; 778 } 779 780 unittest { 781 //test empty string 782 string name = getModuleNameFromContent(""); 783 assert(name == "", "can't get module name from empty string"); 784 785 //test simple name 786 name = getModuleNameFromContent("module myPackage.myModule;"); 787 assert(name == "myPackage.myModule", "can't parse module name"); 788 789 //test if it can ignore module inside comments 790 name = getModuleNameFromContent("/** 791 module fakePackage.fakeModule; 792 */ 793 module myPackage.myModule;"); 794 795 assert(name == "myPackage.myModule", "can't parse module name"); 796 797 name = getModuleNameFromContent("//module fakePackage.fakeModule; 798 module myPackage.myModule;"); 799 800 assert(name == "myPackage.myModule", "can't parse module name"); 801 } 802 803 /** 804 * Search for module keyword in file 805 */ 806 string getModuleNameFromFile(string filePath) { 807 string fileContent = filePath.readText; 808 809 logDiagnostic("Get module name from path: " ~ filePath); 810 return getModuleNameFromContent(fileContent); 811 } 812 813 enum UpgradeOptions 814 { 815 none = 0, 816 upgrade = 1<<1, /// Upgrade existing packages 817 preRelease = 1<<2, /// inclde pre-release versions in upgrade 818 forceRemove = 1<<3, /// Force removing package folders, which contain unknown files 819 select = 1<<4, /// Update the dub.selections.json file with the upgraded versions 820 printUpgradesOnly = 1<<5, /// Instead of downloading new packages, just print a message to notify the user of their existence 821 useCachedResult = 1<<6, /// Use cached information stored with the package to determine upgrades 822 } 823 824 class DependencyVersionResolver : DependencyResolver!(Dependency, Dependency) { 825 protected { 826 Dub m_dub; 827 UpgradeOptions m_options; 828 Dependency[][string] m_packageVersions; 829 Package[string] m_remotePackages; 830 SelectedVersions m_selectedVersions; 831 Package m_rootPackage; 832 } 833 834 835 this(Dub dub, UpgradeOptions options) 836 { 837 m_dub = dub; 838 m_options = options; 839 } 840 841 Dependency[string] resolve(Package root, SelectedVersions selected_versions) 842 { 843 m_rootPackage = root; 844 m_selectedVersions = selected_versions; 845 return super.resolve(TreeNode(root.name, Dependency(root.ver)), (m_options & UpgradeOptions.printUpgradesOnly) == 0); 846 } 847 848 protected override Dependency[] getAllConfigs(string pack) 849 { 850 if (auto pvers = pack in m_packageVersions) 851 return *pvers; 852 853 if (!(m_options & UpgradeOptions.upgrade) && m_selectedVersions.hasSelectedVersion(pack)) { 854 auto ret = [m_selectedVersions.getSelectedVersion(pack)]; 855 logDiagnostic("Using fixed selection %s %s", pack, ret[0]); 856 m_packageVersions[pack] = ret; 857 return ret; 858 } 859 860 logDiagnostic("Search for versions of %s (%s package suppliers)", pack, m_dub.m_packageSuppliers.length); 861 Version[] versions; 862 foreach (p; m_dub.packageManager.getPackageIterator(pack)) 863 versions ~= p.ver; 864 865 foreach (ps; m_dub.m_packageSuppliers) { 866 try { 867 auto vers = ps.getVersions(pack); 868 vers.reverse(); 869 if (!vers.length) { 870 logDiagnostic("No versions for %s for %s", pack, ps.description); 871 continue; 872 } 873 874 versions ~= vers; 875 break; 876 } catch (Exception e) { 877 logDebug("Package %s not found in %s: %s", pack, ps.description, e.msg); 878 logDebug("Full error: %s", e.toString().sanitize); 879 } 880 } 881 882 // sort by version, descending, and remove duplicates 883 versions = versions.sort!"a>b".uniq.array; 884 885 // move pre-release versions to the back of the list if no preRelease flag is given 886 if (!(m_options & UpgradeOptions.preRelease)) 887 versions = versions.filter!(v => !v.isPreRelease).array ~ versions.filter!(v => v.isPreRelease).array; 888 889 if (!versions.length) logDiagnostic("Nothing found for %s", pack); 890 891 auto ret = versions.map!(v => Dependency(v)).array; 892 m_packageVersions[pack] = ret; 893 return ret; 894 } 895 896 protected override Dependency[] getSpecificConfigs(TreeNodes nodes) 897 { 898 if (!nodes.configs.path.empty) return [nodes.configs]; 899 else return null; 900 } 901 902 903 protected override TreeNodes[] getChildren(TreeNode node) 904 { 905 auto ret = appender!(TreeNodes[]); 906 auto pack = getPackage(node.pack, node.config); 907 if (!pack) { 908 // this can hapen when the package description contains syntax errors 909 logDebug("Invalid package in dependency tree: %s %s", node.pack, node.config); 910 return null; 911 } 912 auto basepack = pack.basePackage; 913 914 foreach (dname, dspec; pack.dependencies) { 915 auto dbasename = getBasePackageName(dname); 916 917 // detect dependencies to the root package (or sub packages thereof) 918 if (dbasename == basepack.name) { 919 auto absdeppath = dspec.mapToPath(pack.path).path; 920 auto subpack = m_dub.m_packageManager.getSubPackage(basepack, getSubPackageName(dname), true); 921 if (subpack) { 922 auto desireddeppath = dname == dbasename ? basepack.path : subpack.path; 923 enforce(dspec.path.empty || absdeppath == desireddeppath, 924 format("Dependency from %s to root package references wrong path: %s vs. %s", 925 node.pack, absdeppath.toNativeString(), desireddeppath.toNativeString())); 926 } 927 ret ~= TreeNodes(dname, node.config); 928 continue; 929 } 930 931 if (dspec.optional && !m_dub.packageManager.getFirstPackage(dname)) 932 continue; 933 if (m_options & UpgradeOptions.upgrade || !m_selectedVersions || !m_selectedVersions.hasSelectedVersion(dbasename)) 934 ret ~= TreeNodes(dname, dspec.mapToPath(pack.path)); 935 else ret ~= TreeNodes(dname, m_selectedVersions.getSelectedVersion(dbasename)); 936 } 937 return ret.data; 938 } 939 940 protected override bool matches(Dependency configs, Dependency config) 941 { 942 if (!configs.path.empty) return configs.path == config.path; 943 return configs.merge(config).valid; 944 } 945 946 private Package getPackage(string name, Dependency dep) 947 { 948 auto basename = getBasePackageName(name); 949 950 // for sub packages, first try to get them from the base package 951 if (basename != name) { 952 auto subname = getSubPackageName(name); 953 auto basepack = getPackage(basename, dep); 954 if (!basepack) return null; 955 if (auto sp = m_dub.m_packageManager.getSubPackage(basepack, subname, true)) { 956 return sp; 957 } else if (!basepack.subPackages.canFind!(p => p.path.length)) { 958 // note: external sub packages are handled further below 959 logDiagnostic("Sub package %s doesn't exist in %s %s.", name, basename, dep.version_); 960 return null; 961 } else if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) { 962 return ret; 963 } else { 964 logDiagnostic("External sub package %s %s not found.", name, dep.version_); 965 return null; 966 } 967 } 968 969 if (!dep.path.empty) { 970 auto ret = m_dub.packageManager.getOrLoadPackage(dep.path); 971 if (dep.matches(ret.ver)) return ret; 972 } 973 974 if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) 975 return ret; 976 977 auto key = name ~ ":" ~ dep.version_.toString(); 978 if (auto ret = key in m_remotePackages) 979 return *ret; 980 981 auto prerelease = (m_options & UpgradeOptions.preRelease) != 0; 982 983 auto rootpack = name.split(":")[0]; 984 985 foreach (ps; m_dub.m_packageSuppliers) { 986 if (rootpack == name) { 987 try { 988 auto desc = ps.getPackageDescription(name, dep, prerelease); 989 auto ret = new Package(desc); 990 m_remotePackages[key] = ret; 991 return ret; 992 } catch (Exception e) { 993 logDiagnostic("Metadata for %s %s could not be downloaded from %s: %s", name, dep, ps.description, e.msg); 994 logDebug("Full error: %s", e.toString().sanitize); 995 } 996 } else { 997 logDiagnostic("Package %s not found in base package description (%s). Downloading whole package.", name, dep.version_.toString()); 998 try { 999 FetchOptions fetchOpts; 1000 fetchOpts |= prerelease ? FetchOptions.usePrerelease : FetchOptions.none; 1001 fetchOpts |= (m_options & UpgradeOptions.forceRemove) != 0 ? FetchOptions.forceRemove : FetchOptions.none; 1002 m_dub.fetch(rootpack, dep, defaultPlacementLocation, fetchOpts, "need sub package description"); 1003 auto ret = m_dub.m_packageManager.getBestPackage(name, dep); 1004 if (!ret) { 1005 logWarn("Package %s %s doesn't have a sub package %s", rootpack, dep.version_, name); 1006 return null; 1007 } 1008 m_remotePackages[key] = ret; 1009 return ret; 1010 } catch (Exception e) { 1011 logDiagnostic("Package %s could not be downloaded from %s: %s", rootpack, ps.description, e.msg); 1012 logDebug("Full error: %s", e.toString().sanitize); 1013 } 1014 } 1015 } 1016 1017 m_remotePackages[key] = null; 1018 1019 logWarn("Package %s %s could not be loaded either locally, or from the configured package registries.", name, dep); 1020 return null; 1021 } 1022 }