1 /**
2 	A package manager.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff, 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff, Sönke Ludwig
7 */
8 module dub.dub;
9 
10 import dub.compilers.compiler;
11 import dub.dependency;
12 import dub.dependencyresolver;
13 import dub.internal.utils;
14 import dub.internal.vibecompat.core.file;
15 import dub.internal.vibecompat.core.log;
16 import dub.internal.vibecompat.data.json;
17 import dub.internal.vibecompat.inet.url;
18 import dub.package_;
19 import dub.packagemanager;
20 import dub.packagesupplier;
21 import dub.project;
22 import dub.generators.generator;
23 import dub.init;
24 
25 
26 // todo: cleanup imports.
27 import std.algorithm;
28 import std.array;
29 import std.conv;
30 import std.datetime;
31 import std.exception;
32 import std.file;
33 import std.process;
34 import std.string;
35 import std.typecons;
36 import std.zip;
37 import std.encoding : sanitize;
38 
39 // Workaround for libcurl liker errors when building with LDC
40 version (LDC) pragma(lib, "curl");
41 
42 // Set output path and options for coverage reports
43 version (DigitalMars) version (D_Coverage) static if (__VERSION__ >= 2068)
44 {
45 	shared static this()
46 	{
47 		import core.runtime, std.file, std.path, std.stdio;
48 		dmd_coverSetMerge(true);
49 		auto path = buildPath(dirName(thisExePath()), "../cov");
50 		if (!path.exists)
51 			mkdir(path);
52 		dmd_coverDestPath(path);
53 	}
54 }
55 
56 static this()
57 {
58 	import dub.compilers.dmd : DMDCompiler;
59 	import dub.compilers.gdc : GDCCompiler;
60 	import dub.compilers.ldc : LDCCompiler;
61 	registerCompiler(new DMDCompiler);
62 	registerCompiler(new GDCCompiler);
63 	registerCompiler(new LDCCompiler);
64 }
65 
66 /// The URL to the official package registry.
67 enum defaultRegistryURL = "http://code.dlang.org/";
68 
69 /** Returns a default list of package suppliers.
70 
71 	This will contain a single package supplier that points to the official
72 	package registry.
73 
74 	See_Also: `defaultRegistryURL`
75 */
76 PackageSupplier[] defaultPackageSuppliers()
77 {
78 	logDiagnostic("Using dub registry url '%s'", defaultRegistryURL);
79 	return [new RegistryPackageSupplier(URL(defaultRegistryURL))];
80 }
81 
82 
83 /** Provides a high-level entry point for DUB's functionality.
84 
85 	This class provides means to load a certain project (a root package with
86 	all of its dependencies) and to perform high-level operations as found in
87 	the command line interface.
88 */
89 class Dub {
90 	private {
91 		bool m_dryRun = false;
92 		PackageManager m_packageManager;
93 		PackageSupplier[] m_packageSuppliers;
94 		Path m_rootPath;
95 		SpecialDirs m_dirs;
96 		DubConfig m_config;
97 		Path m_projectPath;
98 		Project m_project;
99 		Path m_overrideSearchPath;
100 		string m_defaultCompiler;
101 	}
102 
103 	/** The default placement location of fetched packages.
104 
105 		This property can be altered, so that packages which are downloaded as part
106 		of the normal upgrade process are stored in a certain location. This is
107 		how the "--local" and "--system" command line switches operate.
108 	*/
109 	PlacementLocation defaultPlacementLocation = PlacementLocation.user;
110 
111 
112 	/** Initializes the instance for use with a specific root package.
113 
114 		Note that a package still has to be loaded using one of the
115 		`loadPackage` overloads.
116 
117 		Params:
118 			root_path = Path to the root package
119 			additional_package_suppliers = A list of package suppliers to try
120 				before the suppliers found in the configurations files and the
121 				`defaultPackageSuppliers`.
122 			skip_registry = Can be used to skip using the configured package
123 				suppliers, as well as the default suppliers.
124 	*/
125 	this(string root_path = ".", PackageSupplier[] additional_package_suppliers = null,
126 			SkipPackageSuppliers skip_registry = SkipPackageSuppliers.none)
127 	{
128 		m_rootPath = Path(root_path);
129 		if (!m_rootPath.absolute) m_rootPath = Path(getcwd()) ~ m_rootPath;
130 
131 		init();
132 
133 		PackageSupplier[] ps = additional_package_suppliers;
134 
135 		if (skip_registry < SkipPackageSuppliers.all)
136 			ps ~= m_config.registryURLs
137 				.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url)))
138 				.array;
139 
140 		if (skip_registry < SkipPackageSuppliers.standard)
141 			ps ~= defaultPackageSuppliers();
142 
143 		m_packageSuppliers = ps;
144 		m_packageManager = new PackageManager(m_dirs.userSettings, m_dirs.systemSettings);
145 		updatePackageSearchPath();
146 	}
147 
148 	/** Initializes the instance with a single package search path, without
149 		loading a package.
150 
151 		This constructor corresponds to the "--bare" option of the command line
152 		interface. Use 
153 	*/
154 	this(Path override_path)
155 	{
156 		init();
157 		m_overrideSearchPath = override_path;
158 		m_packageManager = new PackageManager(Path(), Path(), false);
159 		updatePackageSearchPath();
160 	}
161 
162 	private void init()
163 	{
164 		import std.file : tempDir;
165 		version(Windows) {
166 			m_dirs.systemSettings = Path(environment.get("ProgramData")) ~ "dub/";
167 			m_dirs.userSettings = Path(environment.get("APPDATA")) ~ "dub/";
168 		} else version(Posix){
169 			m_dirs.systemSettings = Path("/var/lib/dub/");
170 			m_dirs.userSettings = Path(environment.get("HOME")) ~ ".dub/";
171 			if (!m_dirs.userSettings.absolute)
172 				m_dirs.userSettings = Path(getcwd()) ~ m_dirs.userSettings;
173 		}
174 
175 		m_dirs.temp = Path(tempDir);
176 
177 		m_config = new DubConfig(jsonFromFile(m_dirs.systemSettings ~ "settings.json", true), m_config);
178 		m_config = new DubConfig(jsonFromFile(Path(thisExePath).parentPath ~ "../etc/dub/settings.json", true), m_config);
179 		m_config = new DubConfig(jsonFromFile(m_dirs.userSettings ~ "settings.json", true), m_config);
180 
181 		determineDefaultCompiler();
182 	}
183 
184 	@property void dryRun(bool v) { m_dryRun = v; }
185 
186 	/** Returns the root path (usually the current working directory).
187 	*/
188 	@property Path rootPath() const { return m_rootPath; }
189 	/// ditto
190 	@property void rootPath(Path root_path)
191 	{
192 		m_rootPath = root_path;
193 		if (!m_rootPath.absolute) m_rootPath = Path(getcwd()) ~ m_rootPath;
194 	}
195 
196 	/// Returns the name listed in the dub.json of the current
197 	/// application.
198 	@property string projectName() const { return m_project.name; }
199 
200 	@property Path projectPath() const { return m_projectPath; }
201 
202 	@property string[] configurations() const { return m_project.configurations; }
203 
204 	@property inout(PackageManager) packageManager() inout { return m_packageManager; }
205 
206 	@property inout(Project) project() inout { return m_project; }
207 
208 	/** Returns the default compiler binary to use for building D code.
209 
210 		If set, the "defaultCompiler" field of the DUB user or system
211 		configuration file will be used. Otherwise the PATH environment variable
212 		will be searched for files named "dmd", "gdc", "gdmd", "ldc2", "ldmd2"
213 		(in that order, taking into account operating system specific file
214 		extensions) and the first match is returned. If no match is found, "dmd"
215 		will be used.
216 	*/
217 	@property string defaultCompiler() const { return m_defaultCompiler; }
218 
219 	/** Loads the package that resides within the configured `rootPath`.
220 	*/
221 	void loadPackage()
222 	{
223 		loadPackage(m_rootPath);
224 	}
225 
226 	/// Loads the package from the specified path as the main project package.
227 	void loadPackage(Path path)
228 	{
229 		m_projectPath = path;
230 		updatePackageSearchPath();
231 		m_project = new Project(m_packageManager, m_projectPath);
232 	}
233 
234 	/// Loads a specific package as the main project package (can be a sub package)
235 	void loadPackage(Package pack)
236 	{
237 		m_projectPath = pack.path;
238 		updatePackageSearchPath();
239 		m_project = new Project(m_packageManager, pack);
240 	}
241 
242 	/** Loads a single file package.
243 
244 		Single-file packages are D files that contain a package receipe comment
245 		at their top. A recipe comment must be a nested `/+ ... +/` style
246 		comment, containing the virtual recipe file name and a colon, followed by the 
247 		recipe contents (what would normally be in dub.sdl/dub.json).
248 
249 		Example:
250 		---
251 		/+ dub.sdl:
252 		   name "test"
253 		   dependency "vibe-d" version="~>0.7.29"
254 		+/
255 		import vibe.http.server;
256 
257 		void main()
258 		{
259 			auto settings = new HTTPServerSettings;
260 			settings.port = 8080;
261 			listenHTTP(settings, &hello);
262 		}
263 
264 		void hello(HTTPServerRequest req, HTTPServerResponse res)
265 		{
266 			res.writeBody("Hello, World!");
267 		}
268 		---
269 
270 		The script above can be invoked with "dub --single test.d".
271 	*/
272 	void loadSingleFilePackage(Path path)
273 	{
274 		import dub.recipe.io : parsePackageRecipe;
275 		import std.file : mkdirRecurse, readText;
276 
277 		path = makeAbsolute(path);
278 
279 		string file_content = readText(path.toNativeString());
280 
281 		if (file_content.startsWith("#!")) {
282 			auto idx = file_content.indexOf('\n');
283 			enforce(idx > 0, "The source fine doesn't contain anything but a shebang line.");
284 			file_content = file_content[idx+1 .. $];
285 		}
286 
287 		file_content = file_content.strip();
288 
289 		string recipe_content;
290 
291 		if (file_content.startsWith("/+")) {
292 			file_content = file_content[2 .. $];
293 			auto idx = file_content.indexOf("+/");
294 			enforce(idx >= 0, "Missing \"+/\" to close comment.");
295 			recipe_content = file_content[0 .. idx].strip();
296 		} else throw new Exception("The source file must start with a recipe comment.");
297 
298 		auto idx = recipe_content.indexOf(':');
299 		enforce(idx > 0, "Missing recipe file name (e.g. \"dub.sdl:\") in recipe comment");
300 		auto recipe_filename = recipe_content[0 .. idx];
301 		recipe_content = recipe_content[idx+1 .. $];
302 
303 		auto recipe = parsePackageRecipe(recipe_content, recipe_filename);
304 		enforce(recipe.buildSettings.sourceFiles.length == 0, "Single-file packages are not allowed to specify source files.");
305 		enforce(recipe.buildSettings.sourcePaths.length == 0, "Single-file packages are not allowed to specify source paths.");
306 		enforce(recipe.buildSettings.importPaths.length == 0, "Single-file packages are not allowed to specify import paths.");
307 		recipe.buildSettings.sourceFiles[""] = [path.toNativeString()];
308 		recipe.buildSettings.sourcePaths[""] = [];
309 		recipe.buildSettings.importPaths[""] = [];
310 		recipe.buildSettings.mainSourceFile = path.toNativeString();
311 		if (recipe.buildSettings.targetType == TargetType.autodetect)
312 			recipe.buildSettings.targetType = TargetType.executable;
313 
314 		auto pack = new Package(recipe, path.parentPath, null, "~master");
315 		loadPackage(pack);
316 	}
317 	/// ditto
318 	void loadSingleFilePackage(string path)
319 	{
320 		loadSingleFilePackage(Path(path));
321 	}
322 
323 	/** Disables the default search paths and only searches a specific directory
324 		for packages.
325 	*/
326 	void overrideSearchPath(Path path)
327 	{
328 		if (!path.absolute) path = Path(getcwd()) ~ path;
329 		m_overrideSearchPath = path;
330 		updatePackageSearchPath();
331 	}
332 
333 	/** Gets the default configuration for a particular build platform.
334 
335 		This forwards to `Project.getDefaultConfiguration` and requires a
336 		project to be loaded.
337 	*/
338 	string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); }
339 
340 	/** Attempts to upgrade the dependency selection of the loaded project.
341 	*/
342 	void upgrade(UpgradeOptions options)
343 	{
344 		// clear non-existent version selections
345 		if (!(options & UpgradeOptions.upgrade)) {
346 			next_pack:
347 			foreach (p; m_project.selections.selectedPackages) {
348 				auto dep = m_project.selections.getSelectedVersion(p);
349 				if (!dep.path.empty) {
350 					auto path = dep.path;
351 					if (!path.absolute) path = this.rootPath ~ path;
352 					try if (m_packageManager.getOrLoadPackage(path)) continue;
353 					catch (Exception e) { logDebug("Failed to load path based selection: %s", e.toString().sanitize); }
354 				} else {
355 					if (m_packageManager.getPackage(p, dep.version_)) continue;
356 					foreach (ps; m_packageSuppliers) {
357 						try {
358 							auto versions = ps.getVersions(p);
359 							if (versions.canFind!(v => dep.matches(v)))
360 								continue next_pack;
361 						} catch (Exception e) {
362 							logDiagnostic("Error querying versions for %s, %s: %s", p, ps.description, e.msg);
363 							logDebug("Full error: %s", e.toString().sanitize());
364 						}
365 					}
366 				}
367 
368 				logWarn("Selected package %s %s doesn't exist. Using latest matching version instead.", p, dep);
369 				m_project.selections.deselectVersion(p);
370 			}
371 		}
372 
373 		Dependency[string] versions;
374 		if ((options & UpgradeOptions.useCachedResult) && m_project.isUpgradeCacheUpToDate()) {
375 			logDiagnostic("Using cached upgrade results...");
376 			versions = m_project.getUpgradeCache();
377 		} else {
378 			auto resolver = new DependencyVersionResolver(this, options);
379 			versions = resolver.resolve(m_project.rootPackage, m_project.selections);
380 			if (options & UpgradeOptions.useCachedResult) {
381 				logDiagnostic("Caching upgrade results...");
382 				m_project.setUpgradeCache(versions);
383 			}
384 		}
385 
386 		if (options & UpgradeOptions.printUpgradesOnly) {
387 			bool any = false;
388 			string rootbasename = getBasePackageName(m_project.rootPackage.name);
389 
390 			foreach (p, ver; versions) {
391 				if (!ver.path.empty) continue;
392 
393 				auto basename = getBasePackageName(p);
394 				if (basename == rootbasename) continue;
395 
396 				if (!m_project.selections.hasSelectedVersion(basename)) {
397 					logInfo("Non-selected package %s is available with version %s.",
398 						basename, ver);
399 					any = true;
400 					continue;
401 				}
402 				auto sver = m_project.selections.getSelectedVersion(basename);
403 				if (!sver.path.empty) continue;
404 				if (ver.version_ <= sver.version_) continue;
405 				logInfo("Package %s can be upgraded from %s to %s.",
406 					basename, sver, ver);
407 				any = true;
408 			}
409 			if (any) logInfo("Use \"dub upgrade\" to perform those changes.");
410 			return;
411 		}
412 
413 		foreach (p; versions.byKey) {
414 			auto ver = versions[p]; // Workaround for DMD 2.070.0 AA issue (crashes in aaApply2 if iterating by key+value)
415 			assert(!p.canFind(":"), "Resolved packages contain a sub package!?: "~p);
416 			Package pack;
417 			if (!ver.path.empty) {
418 				try pack = m_packageManager.getOrLoadPackage(ver.path);
419 				catch (Exception e) {
420 					logDebug("Failed to load path based selection: %s", e.toString().sanitize);
421 					continue;
422 				}
423 			} else {
424 				pack = m_packageManager.getBestPackage(p, ver);
425 				if (pack && m_packageManager.isManagedPackage(pack)
426 					&& ver.version_.isBranch && (options & UpgradeOptions.upgrade) != 0)
427 				{
428 					// TODO: only re-install if there is actually a new commit available
429 					logInfo("Re-installing branch based dependency %s %s", p, ver.toString());
430 					m_packageManager.remove(pack, (options & UpgradeOptions.forceRemove) != 0);
431 					pack = null;
432 				}
433 			}
434 
435 			FetchOptions fetchOpts;
436 			fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none;
437 			fetchOpts |= (options & UpgradeOptions.forceRemove) != 0 ? FetchOptions.forceRemove : FetchOptions.none;
438 			if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version");
439 			if ((options & UpgradeOptions.select) && p != m_project.rootPackage.name) {
440 				if (ver.path.empty) m_project.selections.selectVersion(p, ver.version_);
441 				else {
442 					Path relpath = ver.path;
443 					if (relpath.absolute) relpath = relpath.relativeTo(m_project.rootPackage.path);
444 					m_project.selections.selectVersion(p, relpath);
445 				}
446 			}
447 		}
448 
449 		m_project.reinit();
450 
451 		if ((options & UpgradeOptions.select) && !(options & UpgradeOptions.noSaveSelections))
452 			m_project.saveSelections();
453 	}
454 
455 	/** Generate project files for a specified generator.
456 
457 		Any existing project files will be overridden.
458 	*/
459 	void generateProject(string ide, GeneratorSettings settings)
460 	{
461 		auto generator = createProjectGenerator(ide, m_project);
462 		if (m_dryRun) return; // TODO: pass m_dryRun to the generator
463 		generator.generate(settings);
464 	}
465 
466 	/** Executes tests on the current project.
467 
468 		Throws an exception, if unittests failed.
469 	*/
470 	void testProject(GeneratorSettings settings, string config, Path custom_main_file)
471 	{
472 		if (custom_main_file.length && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file;
473 
474 		if (config.length == 0) {
475 			// if a custom main file was given, favor the first library configuration, so that it can be applied
476 			if (custom_main_file.length) config = m_project.getDefaultConfiguration(settings.platform, false);
477 			// else look for a "unittest" configuration
478 			if (!config.length && m_project.rootPackage.configurations.canFind("unittest")) config = "unittest";
479 			// if not found, fall back to the first "library" configuration
480 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, false);
481 			// if still nothing found, use the first executable configuration
482 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, true);
483 		}
484 
485 		auto generator = createProjectGenerator("build", m_project);
486 
487 		auto test_config = format("__test__%s__", config);
488 
489 		BuildSettings lbuildsettings = settings.buildSettings;
490 		m_project.addBuildSettings(lbuildsettings, settings.platform, config, null, true);
491 		if (lbuildsettings.targetType == TargetType.none) {
492 			logInfo(`Configuration '%s' has target type "none". Skipping test.`, config);
493 			return;
494 		}
495 
496 		if (lbuildsettings.targetType == TargetType.executable) {
497 			if (config == "unittest") logInfo("Running custom 'unittest' configuration.", config);
498 			else logInfo(`Configuration '%s' does not output a library. Falling back to "dub -b unittest -c %s".`, config, config);
499 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
500 			settings.config = config;
501 		} else if (lbuildsettings.sourceFiles.empty) {
502 			logInfo(`No source files found in configuration '%s'. Falling back to "dub -b unittest".`, config);
503 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
504 			settings.config = m_project.getDefaultConfiguration(settings.platform);
505 		} else {
506 			import std.algorithm : remove;
507 
508 			logInfo(`Generating test runner configuration '%s' for '%s' (%s).`, test_config, config, lbuildsettings.targetType);
509 
510 			BuildSettingsTemplate tcinfo = m_project.rootPackage.recipe.getConfiguration(config).buildSettings;
511 			tcinfo.targetType = TargetType.executable;
512 			tcinfo.targetName = test_config;
513 			// HACK for vibe.d's legacy main() behavior:
514 			tcinfo.versions[""] ~= "VibeCustomMain";
515 			m_project.rootPackage.recipe.buildSettings.versions[""] = m_project.rootPackage.recipe.buildSettings.versions.get("", null).remove!(v => v == "VibeDefaultMain");
516 			// TODO: remove this ^ once vibe.d has removed the default main implementation
517 			string custommodname;
518 			if (custom_main_file.length) {
519 				import std.path;
520 				tcinfo.sourceFiles[""] ~= custom_main_file.relativeTo(m_project.rootPackage.path).toNativeString();
521 				tcinfo.importPaths[""] ~= custom_main_file.parentPath.toNativeString();
522 				custommodname = custom_main_file.head.toString().baseName(".d");
523 			}
524 
525 			string[] import_modules;
526 			foreach (file; lbuildsettings.sourceFiles) {
527 				if (file.endsWith(".d") && Path(file).head.toString() != "package.d")
528 					import_modules ~= dub.internal.utils.determineModuleName(lbuildsettings, Path(file), m_project.rootPackage.path);
529 			}
530 
531 			// generate main file
532 			Path mainfile = getTempFile("dub_test_root", ".d");
533 			tcinfo.sourceFiles[""] ~= mainfile.toNativeString();
534 			tcinfo.mainSourceFile = mainfile.toNativeString();
535 			if (!m_dryRun) {
536 				auto fil = openFile(mainfile, FileMode.createTrunc);
537 				scope(exit) fil.close();
538 				fil.write("module dub_test_root;\n");
539 				fil.write("import std.typetuple;\n");
540 				foreach (mod; import_modules) fil.write(format("static import %s;\n", mod));
541 				fil.write("alias allModules = TypeTuple!(");
542 				foreach (i, mod; import_modules) {
543 					if (i > 0) fil.write(", ");
544 					fil.write(mod);
545 				}
546 				fil.write(");\n");
547 				if (custommodname.length) {
548 					fil.write(format("import %s;\n", custommodname));
549 				} else {
550 					fil.write(q{
551 						import std.stdio;
552 						import core.runtime;
553 
554 						void main() { writeln("All unit tests have been run successfully."); }
555 						shared static this() {
556 							version (Have_tested) {
557 								import tested;
558 								import core.runtime;
559 								import std.exception;
560 								Runtime.moduleUnitTester = () => true;
561 								//runUnitTests!app(new JsonTestResultWriter("results.json"));
562 								enforce(runUnitTests!allModules(new ConsoleTestResultWriter), "Unit tests failed.");
563 							}
564 						}
565 					});
566 				}
567 			}
568 			m_project.rootPackage.recipe.configurations ~= ConfigurationInfo(test_config, tcinfo);
569 			m_project = new Project(m_packageManager, m_project.rootPackage);
570 
571 			settings.config = test_config;
572 		}
573 
574 		generator.generate(settings);
575 	}
576 
577 	/** Prints the specified build settings necessary for building the root package.
578 	*/
579 	void listProjectData(GeneratorSettings settings, string[] requestedData, ListBuildSettingsFormat list_type)
580 	{
581 		import std.stdio;
582 		import std.ascii : newline;
583 
584 		// Split comma-separated lists
585 		string[] requestedDataSplit =
586 			requestedData
587 			.map!(a => a.splitter(",").map!strip)
588 			.joiner()
589 			.array();
590 
591 		auto data = m_project.listBuildSettings(settings, requestedDataSplit, list_type);
592 
593 		string delimiter;
594 		final switch (list_type) with (ListBuildSettingsFormat) {
595 			case list: delimiter = newline ~ newline; break;
596 			case listNul: delimiter = "\0\0"; break;
597 			case commandLine: delimiter = " "; break;
598 			case commandLineNul: delimiter = "\0\0"; break;
599 		}
600 
601 		write(data.joiner(delimiter));
602 		if (delimiter != "\0\0") writeln();
603 	}
604 
605 	/// Cleans intermediate/cache files of the given package
606 	void cleanPackage(Path path)
607 	{
608 		logInfo("Cleaning package at %s...", path.toNativeString());
609 		enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString());
610 
611 		// TODO: clear target files and copy files
612 
613 		if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString());
614 		if (existsFile(path ~ ".dub/obj")) rmdirRecurse((path ~ ".dub/obj").toNativeString());
615 	}
616 
617 	/// Fetches the package matching the dependency and places it in the specified location.
618 	Package fetch(string packageId, const Dependency dep, PlacementLocation location, FetchOptions options, string reason = "")
619 	{
620 		Json pinfo;
621 		PackageSupplier supplier;
622 		foreach(ps; m_packageSuppliers){
623 			try {
624 				pinfo = ps.fetchPackageRecipe(packageId, dep, (options & FetchOptions.usePrerelease) != 0);
625 				supplier = ps;
626 				break;
627 			} catch(Exception e) {
628 				logDiagnostic("Package %s not found for %s: %s", packageId, ps.description, e.msg);
629 				logDebug("Full error: %s", e.toString().sanitize());
630 			}
631 		}
632 		enforce(pinfo.type != Json.Type.undefined, "No package "~packageId~" was found matching the dependency "~dep.toString());
633 		string ver = pinfo["version"].get!string;
634 
635 		Path placement;
636 		final switch (location) {
637 			case PlacementLocation.local: placement = m_rootPath; break;
638 			case PlacementLocation.user: placement = m_dirs.userSettings ~ "packages/"; break;
639 			case PlacementLocation.system: placement = m_dirs.systemSettings ~ "packages/"; break;
640 		}
641 
642 		// always upgrade branch based versions - TODO: actually check if there is a new commit available
643 		Package existing;
644 		try existing = m_packageManager.getPackage(packageId, ver, placement);
645 		catch (Exception e) {
646 			logWarn("Failed to load existing package %s: %s", ver, e.msg);
647 			logDiagnostic("Full error: %s", e.toString().sanitize);
648 		}
649 
650 		if (options & FetchOptions.printOnly) {
651 			if (existing && existing.version_ != Version(ver))
652 				logInfo("A new version for %s is available (%s -> %s). Run \"dub upgrade %s\" to switch.",
653 					packageId, existing.version_, ver, packageId);
654 			return null;
655 		}
656 
657 		if (existing) {
658 			if (!ver.startsWith("~") || !(options & FetchOptions.forceBranchUpgrade) || location == PlacementLocation.local) {
659 				// TODO: support git working trees by performing a "git pull" instead of this
660 				logDiagnostic("Package %s %s (%s) is already present with the latest version, skipping upgrade.",
661 					packageId, ver, placement);
662 				return existing;
663 			} else {
664 				logInfo("Removing %s %s to prepare replacement with a new version.", packageId, ver);
665 				if (!m_dryRun) m_packageManager.remove(existing, (options & FetchOptions.forceRemove) != 0);
666 			}
667 		}
668 
669 		if (reason.length) logInfo("Fetching %s %s (%s)...", packageId, ver, reason);
670 		else logInfo("Fetching %s %s...", packageId, ver);
671 		if (m_dryRun) return null;
672 
673 		logDebug("Acquiring package zip file");
674 
675 		auto clean_package_version = ver[ver.startsWith("~") ? 1 : 0 .. $];
676 		clean_package_version = clean_package_version.replace("+", "_"); // + has special meaning for Optlink
677 		if (!placement.existsFile())
678 			mkdirRecurse(placement.toNativeString());
679 		Path dstpath = placement ~ (packageId ~ "-" ~ clean_package_version);
680 		if (!dstpath.existsFile())
681 			mkdirRecurse(dstpath.toNativeString());
682 
683 		// Support libraries typically used with git submodules like ae.
684 		// Such libraries need to have ".." as import path but this can create
685 		// import path leakage.
686 		dstpath = dstpath ~ packageId;
687 
688 		auto lock = lockFile(dstpath.toNativeString() ~ ".lock", 30.seconds); // possibly wait for other dub instance
689 		if (dstpath.existsFile())
690 		{
691 			m_packageManager.refresh(false);
692 			return m_packageManager.getPackage(packageId, ver, dstpath);
693 		}
694 
695 		auto path = getTempFile(packageId, ".zip");
696 		supplier.fetchPackage(path, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
697 		scope(exit) std.file.remove(path.toNativeString());
698 
699 		logDiagnostic("Placing to %s...", placement.toNativeString());
700 		return m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
701 	}
702 
703 	/** Removes a specific locally cached package.
704 
705 		This will delete the package files from disk and removes the
706 		corresponding entry from the list of known packages.
707 
708 		Params:
709 			pack = Package instance to remove
710 			force_remove = Forces removal of the package, even if untracked
711 				files are found in its folder.
712 	*/
713 	void remove(in Package pack, bool force_remove)
714 	{
715 		logInfo("Removing %s in %s", pack.name, pack.path.toNativeString());
716 		if (!m_dryRun) m_packageManager.remove(pack, force_remove);
717 	}
718 
719 	/// @see remove(string, string, RemoveLocation)
720 	enum RemoveVersionWildcard = "*";
721 
722 	/** Removes one or more versions of a locally cached package.
723 
724 		This will remove a given package with a specified version from the
725 		given location. It will remove at most one package, unless `version_`
726 		is set to `RemoveVersionWildcard`.
727 
728 		Params:
729 			package_id = Name of the package to be removed
730 			location_ = Specifies the location to look for the given package
731 				name/version.
732 			force_remove = Forces removal of the package, even if untracked
733 				files are found in its folder.
734 			resolve_version = Callback to select package version.
735 	*/
736 	void remove(string package_id, PlacementLocation location, bool force_remove,
737 				scope size_t delegate(in Package[] packages) resolve_version)
738 	{
739 		enforce(!package_id.empty);
740 		if (location == PlacementLocation.local) {
741 			logInfo("To remove a locally placed package, make sure you don't have any data"
742 					~ "\nleft in it's directory and then simply remove the whole directory.");
743 			throw new Exception("dub cannot remove locally installed packages.");
744 		}
745 
746 		Package[] packages;
747 
748 		// Retrieve packages to be removed.
749 		foreach(pack; m_packageManager.getPackageIterator(package_id))
750 			if (m_packageManager.isManagedPackage(pack))
751 				packages ~= pack;
752 
753 		// Check validity of packages to be removed.
754 		if(packages.empty) {
755 			throw new Exception("Cannot find package to remove. ("
756 				~ "id: '" ~ package_id ~ "', location: '" ~ to!string(location) ~ "'"
757 				~ ")");
758 		}
759 
760 		immutable idx = resolve_version(packages);
761 		if (idx == size_t.max)
762 			return;
763 		else if (idx != packages.length)
764 			packages = packages[idx .. idx + 1];
765 
766 		logDebug("Removing %s packages.", packages.length);
767 		foreach(pack; packages) {
768 			try {
769 				remove(pack, force_remove);
770 				logInfo("Removed %s, version %s.", package_id, pack.version_);
771 			} catch (Exception e) {
772 				logError("Failed to remove %s %s: %s", package_id, pack.version_, e.msg);
773 				logInfo("Continuing with other packages (if any).");
774 			}
775 		}
776 	}
777 
778 	/** Removes a specific version of a package.
779 
780 		Params:
781 			package_id = Name of the package to be removed
782 			version_ = Identifying a version or a wild card. If an empty string
783 				is passed, the package will be removed from the location, if
784 				there is only one version retrieved. This will throw an
785 				exception, if there are multiple versions retrieved.
786 			location_ = Specifies the location to look for the given package
787 				name/version.
788 			force_remove = Forces removal of the package, even if untracked
789 				files are found in its folder.
790 	 */
791 	void remove(string package_id, string version_, PlacementLocation location, bool force_remove)
792 	{
793 		remove(package_id, location, force_remove, (in packages) {
794 			if (version_ == RemoveVersionWildcard)
795 				return packages.length;
796 			if (version_.empty && packages.length > 1) {
797 				logError("Cannot remove package '" ~ package_id ~ "', there are multiple possibilities at location\n"
798 						 ~ "'" ~ to!string(location) ~ "'.");
799 				logError("Available versions:");
800 				foreach(pack; packages)
801 					logError("  %s", pack.version_);
802 				throw new Exception("Please specify a individual version using --version=... or use the"
803 									~ " wildcard --version=" ~ RemoveVersionWildcard ~ " to remove all versions.");
804 			}
805 			foreach (i, p; packages) {
806 				if (p.version_ == Version(version_))
807 					return i;
808 			}
809 			throw new Exception("Cannot find package to remove. ("
810 				~ "id: '" ~ package_id ~ "', version: '" ~ version_ ~ "', location: '" ~ to!string(location) ~ "'"
811 				~ ")");
812 		});
813 	}
814 
815 	/** Adds a directory to the list of locally known packages.
816 
817 		Forwards to `PackageManager.addLocalPackage`.
818 
819 		Params:
820 			path = Path to the package
821 			ver = Optional version to associate with the package (can be left
822 				empty)
823 			system = Make the package known system wide instead of user wide
824 				(requires administrator privileges).
825 
826 		See_Also: `removeLocalPackage`
827 	*/
828 	void addLocalPackage(string path, string ver, bool system)
829 	{
830 		if (m_dryRun) return;
831 		m_packageManager.addLocalPackage(makeAbsolute(path), ver, system ? LocalPackageType.system : LocalPackageType.user);
832 	}
833 
834 	/** Removes a directory from the list of locally known packages.
835 
836 		Forwards to `PackageManager.removeLocalPackage`.
837 
838 		Params:
839 			path = Path to the package
840 			system = Make the package known system wide instead of user wide
841 				(requires administrator privileges).
842 
843 		See_Also: `addLocalPackage`
844 	*/
845 	void removeLocalPackage(string path, bool system)
846 	{
847 		if (m_dryRun) return;
848 		m_packageManager.removeLocalPackage(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
849 	}
850 
851 	/** Registers a local directory to search for packages to use for satisfying
852 		dependencies.
853 
854 		Params:
855 			path = Path to a directory containing package directories
856 			system = Make the package known system wide instead of user wide
857 				(requires administrator privileges).
858 
859 		See_Also: `removeSearchPath`
860 	*/
861 	void addSearchPath(string path, bool system)
862 	{
863 		if (m_dryRun) return;
864 		m_packageManager.addSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
865 	}
866 
867 	/** Unregisters a local directory search path.
868 
869 		Params:
870 			path = Path to a directory containing package directories
871 			system = Make the package known system wide instead of user wide
872 				(requires administrator privileges).
873 
874 		See_Also: `addSearchPath`
875 	*/
876 	void removeSearchPath(string path, bool system)
877 	{
878 		if (m_dryRun) return;
879 		m_packageManager.removeSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
880 	}
881 
882 	/** Queries all package suppliers with the given query string.
883 
884 		Returns a list of tuples, where the first entry is the human readable
885 		name of the package supplier and the second entry is the list of
886 		matched packages.
887 
888 		See_Also: `PackageSupplier.searchPackages`
889 	*/
890 	auto searchPackages(string query)
891 	{
892 		return m_packageSuppliers.map!(ps => tuple(ps.description, ps.searchPackages(query))).array
893 			.filter!(t => t[1].length);
894 	}
895 
896 	/** Returns a list of all available versions (including branches) for a
897 		particular package.
898 
899 		The list returned is based on the registered package suppliers. Local
900 		packages are not queried in the search for versions.
901 
902 		See_also: `getLatestVersion`
903 	*/
904 	Version[] listPackageVersions(string name)
905 	{
906 		Version[] versions;
907 		foreach (ps; this.m_packageSuppliers) {
908 			try versions ~= ps.getVersions(name);
909 			catch (Exception e) {
910 				logDebug("Failed to get versions for package %s on provider %s: %s", name, ps.description, e.msg);
911 			}
912 		}
913 		return versions.sort().uniq.array;
914 	}
915 
916 	/** Returns the latest available version for a particular package.
917 
918 		This function returns the latest numbered version of a package. If no
919 		numbered versions are available, it will return an available branch,
920 		preferring "~master".
921 
922 		Params:
923 			package_name: The name of the package in question.
924 			prefer_stable: If set to `true` (the default), returns the latest
925 				stable version, even if there are newer pre-release versions.
926 
927 		See_also: `listPackageVersions`
928 	*/
929 	Version getLatestVersion(string package_name, bool prefer_stable = true)
930 	{
931 		auto vers = listPackageVersions(package_name);
932 		enforce(!vers.empty, "Failed to find any valid versions for a package name of '"~package_name~"'.");
933 		auto final_versions = vers.filter!(v => !v.isBranch && !v.isPreRelease).array;
934 		if (prefer_stable && final_versions.length) return final_versions[$-1];
935 		else if (vers[$-1].isBranch) return vers[$-1];
936 		else return vers[$-1];
937 	}
938 
939 	/** Initializes a directory with a package skeleton.
940 
941 		Params:
942 			path = Path of the directory to create the new package in. The
943 				directory will be created if it doesn't exist.
944 			deps = List of dependencies to add to the package recipe.
945 			type = Specifies the type of the application skeleton to use.
946 			format = Determines the package recipe format to use.
947 			recipe_callback = Optional callback that can be used to
948 				customize the recipe before it gets written.
949 	*/
950 	void createEmptyPackage(Path path, string[] deps, string type,
951 		PackageFormat format = PackageFormat.sdl,
952 		scope void delegate(ref PackageRecipe, ref PackageFormat) recipe_callback = null)
953 	{
954 		if (!path.absolute) path = m_rootPath ~ path;
955 		path.normalize();
956 
957 		string[string] depVers;
958 		string[] notFound; // keep track of any failed packages in here
959 		foreach (dep; deps) {
960 			Version ver;
961 			try {
962 				ver = getLatestVersion(dep);
963 				depVers[dep] = ver.isBranch ? ver.toString() : "~>" ~ ver.toString();
964 			} catch (Exception e) {
965 				notFound ~= dep;
966 			}
967 		}
968 
969 		if(notFound.length > 1){
970 			throw new Exception(.format("Couldn't find packages: %-(%s, %).", notFound));
971 		}
972 		else if(notFound.length == 1){
973 			throw new Exception(.format("Couldn't find package: %-(%s, %).", notFound));
974 		}
975 
976 		if (m_dryRun) return;
977 
978 		initPackage(path, depVers, type, format, recipe_callback);
979 
980 		//Act smug to the user.
981 		logInfo("Successfully created an empty project in '%s'.", path.toNativeString());
982 	}
983 
984 	/** Converts the package recipe of the loaded root package to the given format.
985 
986 		Params:
987 			destination_file_ext = The file extension matching the desired
988 				format. Possible values are "json" or "sdl".
989 			print_only = Print the converted recipe instead of writing to disk
990 	*/
991 	void convertRecipe(string destination_file_ext, bool print_only = false)
992 	{
993 		import std.path : extension;
994 		import std.stdio : stdout;
995 		import dub.recipe.io : serializePackageRecipe, writePackageRecipe;
996 
997 		if (print_only) {
998 			auto dst = stdout.lockingTextWriter;
999 			serializePackageRecipe(dst, m_project.rootPackage.rawRecipe, "dub."~destination_file_ext);
1000 			return;
1001 		}
1002 
1003 		auto srcfile = m_project.rootPackage.recipePath;
1004 		auto srcext = srcfile[$-1].toString().extension;
1005 		if (srcext == "."~destination_file_ext) {
1006 			logInfo("Package format is already %s.", destination_file_ext);
1007 			return;
1008 		}
1009 
1010 		writePackageRecipe(srcfile[0 .. $-1] ~ ("dub."~destination_file_ext), m_project.rootPackage.rawRecipe);
1011 		removeFile(srcfile);
1012 	}
1013 
1014 	/** Runs DDOX to generate or serve documentation.
1015 
1016 		Params:
1017 			run = If set to true, serves documentation on a local web server.
1018 				Otherwise generates actual HTML files.
1019 			generate_args = Additional command line arguments to pass to
1020 				"ddox generate-html" or "ddox serve-html".
1021 	*/
1022 	void runDdox(bool run, string[] generate_args = null)
1023 	{
1024 		if (m_dryRun) return;
1025 
1026 		// allow to choose a custom ddox tool
1027 		auto tool = m_project.rootPackage.recipe.ddoxTool;
1028 		if (tool.empty) tool = "ddox";
1029 
1030 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
1031 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
1032 		if (!tool_pack) {
1033 			logInfo("% is not present, getting and storing it user wide", tool);
1034 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1035 		}
1036 
1037 		auto ddox_dub = new Dub(null, m_packageSuppliers);
1038 		ddox_dub.loadPackage(tool_pack.path);
1039 		ddox_dub.upgrade(UpgradeOptions.select);
1040 
1041 		auto compiler_binary = this.defaultCompiler;
1042 
1043 		GeneratorSettings settings;
1044 		settings.config = "application";
1045 		settings.compiler = getCompiler(compiler_binary); // TODO: not using --compiler ???
1046 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary);
1047 		settings.buildType = "debug";
1048 		settings.run = true;
1049 
1050 		auto filterargs = m_project.rootPackage.recipe.ddoxFilterArgs.dup;
1051 		if (filterargs.empty) filterargs = ["--min-protection=Protected", "--only-documented"];
1052 
1053 		settings.runArgs = "filter" ~ filterargs ~ "docs.json";
1054 		ddox_dub.generateProject("build", settings);
1055 
1056 		auto p = tool_pack.path;
1057 		p.endsWithSlash = true;
1058 		auto tool_path = p.toNativeString();
1059 
1060 		if (run) {
1061 			settings.runArgs = ["serve-html", "--navigation-type=ModuleTree", "docs.json", "--web-file-dir="~tool_path~"public"] ~ generate_args;
1062 			browse("http://127.0.0.1:8080/");
1063 		} else {
1064 			settings.runArgs = ["generate-html", "--navigation-type=ModuleTree", "docs.json", "docs"] ~ generate_args;
1065 		}
1066 		ddox_dub.generateProject("build", settings);
1067 
1068 		if (!run) {
1069 			// TODO: ddox should copy those files itself
1070 			version(Windows) runCommand("xcopy /S /D "~tool_path~"public\\* docs\\");
1071 			else runCommand("rsync -ru '"~tool_path~"public/' docs/");
1072 		}
1073 	}
1074 
1075 	private void updatePackageSearchPath()
1076 	{
1077 		if (m_overrideSearchPath.length) {
1078 			m_packageManager.disableDefaultSearchPaths = true;
1079 			m_packageManager.searchPath = [m_overrideSearchPath];
1080 		} else {
1081 			auto p = environment.get("DUBPATH");
1082 			Path[] paths;
1083 
1084 			version(Windows) enum pathsep = ";";
1085 			else enum pathsep = ":";
1086 			if (p.length) paths ~= p.split(pathsep).map!(p => Path(p))().array();
1087 			m_packageManager.disableDefaultSearchPaths = false;
1088 			m_packageManager.searchPath = paths;
1089 		}
1090 	}
1091 
1092 	private void determineDefaultCompiler()
1093 	{
1094 		import std.process : environment;
1095 
1096 		m_defaultCompiler = m_config.defaultCompiler;
1097 		if (m_defaultCompiler.length) return;
1098 
1099 		version (Windows) enum sep = ";", exe = ".exe";
1100 		version (Posix) enum sep = ":", exe = "";
1101 
1102 		auto compilers = ["dmd", "gdc", "gdmd", "ldc2", "ldmd2"];
1103 
1104 		auto paths = environment.get("PATH", "").splitter(sep).map!Path;
1105 		auto res = compilers.find!(bin => paths.canFind!(p => existsFile(p ~ (bin~exe))));
1106 		m_defaultCompiler = res.empty ? compilers[0] : res.front;
1107 	}
1108 
1109 	private Path makeAbsolute(Path p) const { return p.absolute ? p : m_rootPath ~ p; }
1110 	private Path makeAbsolute(string p) const { return makeAbsolute(Path(p)); }
1111 }
1112 
1113 
1114 /// Option flags for `Dub.fetch`
1115 enum FetchOptions
1116 {
1117 	none = 0,
1118 	forceBranchUpgrade = 1<<0,
1119 	usePrerelease = 1<<1,
1120 	forceRemove = 1<<2,
1121 	printOnly = 1<<3,
1122 }
1123 
1124 /// Option flags for `Dub.upgrade`
1125 enum UpgradeOptions
1126 {
1127 	none = 0,
1128 	upgrade = 1<<1, /// Upgrade existing packages
1129 	preRelease = 1<<2, /// inclde pre-release versions in upgrade
1130 	forceRemove = 1<<3, /// Force removing package folders, which contain unknown files
1131 	select = 1<<4, /// Update the dub.selections.json file with the upgraded versions
1132 	printUpgradesOnly = 1<<5, /// Instead of downloading new packages, just print a message to notify the user of their existence
1133 	useCachedResult = 1<<6, /// Use cached information stored with the package to determine upgrades
1134 	noSaveSelections = 1<<7, /// Don't store updated selections on disk
1135 }
1136 
1137 /// Determines which of the default package suppliers are queried for packages.
1138 enum SkipPackageSuppliers {
1139 	none,     /// Uses all configured package suppliers.
1140 	standard, /// Does not use the default package suppliers (`defaultPackageSuppliers`).
1141 	all       /// Uses only manually specified package suppliers.
1142 }
1143 
1144 private class DependencyVersionResolver : DependencyResolver!(Dependency, Dependency) {
1145 	protected {
1146 		Dub m_dub;
1147 		UpgradeOptions m_options;
1148 		Dependency[][string] m_packageVersions;
1149 		Package[string] m_remotePackages;
1150 		SelectedVersions m_selectedVersions;
1151 		Package m_rootPackage;
1152 	}
1153 
1154 
1155 	this(Dub dub, UpgradeOptions options)
1156 	{
1157 		m_dub = dub;
1158 		m_options = options;
1159 	}
1160 
1161 	Dependency[string] resolve(Package root, SelectedVersions selected_versions)
1162 	{
1163 		m_rootPackage = root;
1164 		m_selectedVersions = selected_versions;
1165 		return super.resolve(TreeNode(root.name, Dependency(root.version_)), (m_options & UpgradeOptions.printUpgradesOnly) == 0);
1166 	}
1167 
1168 	protected override Dependency[] getAllConfigs(string pack)
1169 	{
1170 		if (auto pvers = pack in m_packageVersions)
1171 			return *pvers;
1172 
1173 		if (!(m_options & UpgradeOptions.upgrade) && m_selectedVersions.hasSelectedVersion(pack)) {
1174 			auto ret = [m_selectedVersions.getSelectedVersion(pack)];
1175 			logDiagnostic("Using fixed selection %s %s", pack, ret[0]);
1176 			m_packageVersions[pack] = ret;
1177 			return ret;
1178 		}
1179 
1180 		logDiagnostic("Search for versions of %s (%s package suppliers)", pack, m_dub.m_packageSuppliers.length);
1181 		Version[] versions;
1182 		foreach (p; m_dub.packageManager.getPackageIterator(pack))
1183 			versions ~= p.version_;
1184 
1185 		foreach (ps; m_dub.m_packageSuppliers) {
1186 			try {
1187 				auto vers = ps.getVersions(pack);
1188 				vers.reverse();
1189 				if (!vers.length) {
1190 					logDiagnostic("No versions for %s for %s", pack, ps.description);
1191 					continue;
1192 				}
1193 
1194 				versions ~= vers;
1195 				break;
1196 			} catch (Exception e) {
1197 				logDebug("Package %s not found in %s: %s", pack, ps.description, e.msg);
1198 				logDebug("Full error: %s", e.toString().sanitize);
1199 			}
1200 		}
1201 
1202 		// sort by version, descending, and remove duplicates
1203 		versions = versions.sort!"a>b".uniq.array;
1204 
1205 		// move pre-release versions to the back of the list if no preRelease flag is given
1206 		if (!(m_options & UpgradeOptions.preRelease))
1207 			versions = versions.filter!(v => !v.isPreRelease).array ~ versions.filter!(v => v.isPreRelease).array;
1208 
1209 		// filter out invalid/unreachable dependency specs
1210 		versions = versions.filter!((v) {
1211 				bool valid = getPackage(pack, Dependency(v)) !is null;
1212 				if (!valid) logDiagnostic("Excluding invalid dependency specification %s %s from dependency resolution process.", pack, v);
1213 				return valid;
1214 			}).array;
1215 
1216 		if (!versions.length) logDiagnostic("Nothing found for %s", pack);
1217 		else logDiagnostic("Return for %s: %s", pack, versions);
1218 
1219 		auto ret = versions.map!(v => Dependency(v)).array;
1220 		m_packageVersions[pack] = ret;
1221 		return ret;
1222 	}
1223 
1224 	protected override Dependency[] getSpecificConfigs(string pack, TreeNodes nodes)
1225 	{
1226 		if (!nodes.configs.path.empty && getPackage(pack, nodes.configs)) return [nodes.configs];
1227 		else return null;
1228 	}
1229 
1230 
1231 	protected override TreeNodes[] getChildren(TreeNode node)
1232 	{
1233 		auto ret = appender!(TreeNodes[]);
1234 		auto pack = getPackage(node.pack, node.config);
1235 		if (!pack) {
1236 			// this can hapen when the package description contains syntax errors
1237 			logDebug("Invalid package in dependency tree: %s %s", node.pack, node.config);
1238 			return null;
1239 		}
1240 		auto basepack = pack.basePackage;
1241 
1242 		foreach (d; pack.getAllDependencies()) {
1243 			auto dbasename = getBasePackageName(d.name);
1244 
1245 			// detect dependencies to the root package (or sub packages thereof)
1246 			if (dbasename == basepack.name) {
1247 				auto absdeppath = d.spec.mapToPath(pack.path).path;
1248 				absdeppath.endsWithSlash = true;
1249 				auto subpack = m_dub.m_packageManager.getSubPackage(basepack, getSubPackageName(d.name), true);
1250 				if (subpack) {
1251 					auto desireddeppath = d.name == dbasename ? basepack.path : subpack.path;
1252 					desireddeppath.endsWithSlash = true;
1253 					enforce(d.spec.path.empty || absdeppath == desireddeppath,
1254 						format("Dependency from %s to root package references wrong path: %s vs. %s",
1255 							node.pack, absdeppath.toNativeString(), desireddeppath.toNativeString()));
1256 				}
1257 				ret ~= TreeNodes(d.name, node.config);
1258 				continue;
1259 			}
1260 
1261 			DependencyType dt;
1262 			if (d.spec.optional) {
1263 				if (d.spec.default_) dt = DependencyType.optionalDefault;
1264 				else dt = DependencyType.optional;
1265 			} else dt = DependencyType.required;
1266 
1267 			if (m_options & UpgradeOptions.upgrade || !m_selectedVersions || !m_selectedVersions.hasSelectedVersion(dbasename)) {
1268 				// keep deselected dependencies deselected by default
1269 				if (m_selectedVersions && !m_selectedVersions.bare && dt == DependencyType.optionalDefault)
1270 					dt = DependencyType.optional;
1271 				ret ~= TreeNodes(d.name, d.spec.mapToPath(pack.path), dt);
1272 			} else {
1273 				// keep already selected optional dependencies if possible
1274 				if (dt == DependencyType.optional) dt = DependencyType.optionalDefault;
1275 				ret ~= TreeNodes(d.name, m_selectedVersions.getSelectedVersion(dbasename), dt);
1276 			}
1277 		}
1278 		return ret.data;
1279 	}
1280 
1281 	protected override bool matches(Dependency configs, Dependency config)
1282 	{
1283 		if (!configs.path.empty) return configs.path == config.path;
1284 		return configs.merge(config).valid;
1285 	}
1286 
1287 	private Package getPackage(string name, Dependency dep)
1288 	{
1289 		auto basename = getBasePackageName(name);
1290 
1291 		// for sub packages, first try to get them from the base package
1292 		if (basename != name) {
1293 			auto subname = getSubPackageName(name);
1294 			auto basepack = getPackage(basename, dep);
1295 			if (!basepack) return null;
1296 			if (auto sp = m_dub.m_packageManager.getSubPackage(basepack, subname, true)) {
1297 				return sp;
1298 			} else if (!basepack.subPackages.canFind!(p => p.path.length)) {
1299 				// note: external sub packages are handled further below
1300 				auto spr = basepack.getInternalSubPackage(subname);
1301 				if (!spr.isNull) {
1302 					auto sp = new Package(spr, basepack.path, basepack);
1303 					m_remotePackages[sp.name] = sp;
1304 					return sp;
1305 				} else {
1306 					logDiagnostic("Sub package %s doesn't exist in %s %s.", name, basename, dep.version_);
1307 					return null;
1308 				}
1309 			} else if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) {
1310 				return ret;
1311 			} else {
1312 				logDiagnostic("External sub package %s %s not found.", name, dep.version_);
1313 				return null;
1314 			}
1315 		}
1316 
1317 		// shortcut if the referenced package is the root package
1318 		if (basename == m_rootPackage.basePackage.name)
1319 			return m_rootPackage.basePackage;
1320 
1321 		if (!dep.path.empty) {
1322 			try {
1323 				auto ret = m_dub.packageManager.getOrLoadPackage(dep.path);
1324 				if (dep.matches(ret.version_)) return ret;
1325 			} catch (Exception e) {
1326 				logDiagnostic("Failed to load path based dependency %s: %s", name, e.msg);
1327 				logDebug("Full error: %s", e.toString().sanitize);
1328 				return null;
1329 			}
1330 		}
1331 
1332 		if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep))
1333 			return ret;
1334 
1335 		auto key = name ~ ":" ~ dep.version_.toString();
1336 		if (auto ret = key in m_remotePackages)
1337 			return *ret;
1338 
1339 		auto prerelease = (m_options & UpgradeOptions.preRelease) != 0;
1340 
1341 		auto rootpack = name.split(":")[0];
1342 
1343 		foreach (ps; m_dub.m_packageSuppliers) {
1344 			if (rootpack == name) {
1345 				try {
1346 					auto desc = ps.fetchPackageRecipe(name, dep, prerelease);
1347 					auto ret = new Package(desc);
1348 					m_remotePackages[key] = ret;
1349 					return ret;
1350 				} catch (Exception e) {
1351 					logDiagnostic("Metadata for %s %s could not be downloaded from %s: %s", name, dep, ps.description, e.msg);
1352 					logDebug("Full error: %s", e.toString().sanitize);
1353 				}
1354 			} else {
1355 				logDiagnostic("Package %s not found in base package description (%s). Downloading whole package.", name, dep.version_.toString());
1356 				try {
1357 					FetchOptions fetchOpts;
1358 					fetchOpts |= prerelease ? FetchOptions.usePrerelease : FetchOptions.none;
1359 					fetchOpts |= (m_options & UpgradeOptions.forceRemove) != 0 ? FetchOptions.forceRemove : FetchOptions.none;
1360 					m_dub.fetch(rootpack, dep, m_dub.defaultPlacementLocation, fetchOpts, "need sub package description");
1361 					auto ret = m_dub.m_packageManager.getBestPackage(name, dep);
1362 					if (!ret) {
1363 						logWarn("Package %s %s doesn't have a sub package %s", rootpack, dep.version_, name);
1364 						return null;
1365 					}
1366 					m_remotePackages[key] = ret;
1367 					return ret;
1368 				} catch (Exception e) {
1369 					logDiagnostic("Package %s could not be downloaded from %s: %s", rootpack, ps.description, e.msg);
1370 					logDebug("Full error: %s", e.toString().sanitize);
1371 				}
1372 			}
1373 		}
1374 
1375 		m_remotePackages[key] = null;
1376 
1377 		logWarn("Package %s %s could not be loaded either locally, or from the configured package registries.", name, dep);
1378 		return null;
1379 	}
1380 }
1381 
1382 private struct SpecialDirs {
1383 	Path temp;
1384 	Path userSettings;
1385 	Path systemSettings;
1386 }
1387 
1388 private class DubConfig {
1389 	private {
1390 		DubConfig m_parentConfig;
1391 		Json m_data;
1392 	}
1393 
1394 	this(Json data, DubConfig parent_config)
1395 	{
1396 		m_data = data;
1397 		m_parentConfig = parent_config;
1398 	}
1399 
1400 	@property string[] registryURLs()
1401 	{
1402 		string[] ret;
1403 		if (auto pv = "registryUrls" in m_data)
1404 			ret = (*pv).deserializeJson!(string[]);
1405 		if (m_parentConfig) ret ~= m_parentConfig.registryURLs;
1406 		return ret;
1407 	}
1408 
1409 	@property string defaultCompiler()
1410 	const {
1411 		if (auto pv = "defaultCompiler" in m_data)
1412 			return pv.get!string;
1413 		if (m_parentConfig) return m_parentConfig.defaultCompiler;
1414 		return null;
1415 	}
1416 }