1 /**
2 	A package manager.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff, 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff, Sönke Ludwig
7 */
8 module dub.dub;
9 
10 import dub.compilers.compiler;
11 import dub.dependency;
12 import dub.dependencyresolver;
13 import dub.internal.utils;
14 import dub.internal.vibecompat.core.file;
15 import dub.internal.vibecompat.core.log;
16 import dub.internal.vibecompat.data.json;
17 import dub.internal.vibecompat.inet.url;
18 import dub.package_;
19 import dub.packagemanager;
20 import dub.packagesuppliers;
21 import dub.project;
22 import dub.generators.generator;
23 import dub.init;
24 
25 import std.algorithm;
26 import std.array : array, replace;
27 import std.conv : to;
28 import std.exception : enforce;
29 import std.file;
30 import std.process : environment;
31 import std.range : assumeSorted, empty;
32 import std..string;
33 import std.encoding : sanitize;
34 
35 // Set output path and options for coverage reports
36 version (DigitalMars) version (D_Coverage)
37 {
38 	shared static this()
39 	{
40 		import core.runtime, std.file, std.path, std.stdio;
41 		dmd_coverSetMerge(true);
42 		auto path = buildPath(dirName(thisExePath()), "../cov");
43 		if (!path.exists)
44 			mkdir(path);
45 		dmd_coverDestPath(path);
46 	}
47 }
48 
49 static this()
50 {
51 	import dub.compilers.dmd : DMDCompiler;
52 	import dub.compilers.gdc : GDCCompiler;
53 	import dub.compilers.ldc : LDCCompiler;
54 	registerCompiler(new DMDCompiler);
55 	registerCompiler(new GDCCompiler);
56 	registerCompiler(new LDCCompiler);
57 }
58 
59 deprecated("use defaultRegistryURLs") enum defaultRegistryURL = defaultRegistryURLs[0];
60 
61 /// The URL to the official package registry and it's default fallback registries.
62 static immutable string[] defaultRegistryURLs = [
63 	"https://code.dlang.org/",
64 	"https://codemirror.dlang.org/",
65 	"https://dub.bytecraft.nl/",
66 	"https://code-mirror.dlang.io/",
67 ];
68 
69 /** Returns a default list of package suppliers.
70 
71 	This will contain a single package supplier that points to the official
72 	package registry.
73 
74 	See_Also: `defaultRegistryURLs`
75 */
76 PackageSupplier[] defaultPackageSuppliers()
77 {
78 	logDiagnostic("Using dub registry url '%s'", defaultRegistryURLs[0]);
79 	return [new FallbackPackageSupplier(defaultRegistryURLs.map!getRegistryPackageSupplier.array)];
80 }
81 
82 /** Returns a registry package supplier according to protocol.
83 
84 	Allowed protocols are dub+http(s):// and maven+http(s)://.
85 */
86 PackageSupplier getRegistryPackageSupplier(string url)
87 {
88 	switch (url.startsWith("dub+", "mvn+", "file://"))
89 	{
90 		case 1:
91 			return new RegistryPackageSupplier(URL(url[4..$]));
92 		case 2:
93 			return new MavenRegistryPackageSupplier(URL(url[4..$]));
94 		case 3:
95 			return new FileSystemPackageSupplier(NativePath(url[7..$]));
96 		default:
97 			return new RegistryPackageSupplier(URL(url));
98 	}
99 }
100 
101 unittest
102 {
103 	auto dubRegistryPackageSupplier = getRegistryPackageSupplier("dub+https://code.dlang.org");
104 	assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org"));
105 
106 	dubRegistryPackageSupplier = getRegistryPackageSupplier("https://code.dlang.org");
107 	assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org"));
108 
109 	auto mavenRegistryPackageSupplier = getRegistryPackageSupplier("mvn+http://localhost:8040/maven/libs-release/dubpackages");
110 	assert(mavenRegistryPackageSupplier.description.canFind(" http://localhost:8040/maven/libs-release/dubpackages"));
111 
112 	auto fileSystemPackageSupplier = getRegistryPackageSupplier("file:///etc/dubpackages");
113 	assert(fileSystemPackageSupplier.description.canFind(" " ~ NativePath("/etc/dubpackages").toNativeString));
114 }
115 
116 /** Provides a high-level entry point for DUB's functionality.
117 
118 	This class provides means to load a certain project (a root package with
119 	all of its dependencies) and to perform high-level operations as found in
120 	the command line interface.
121 */
122 class Dub {
123 	private {
124 		bool m_dryRun = false;
125 		PackageManager m_packageManager;
126 		PackageSupplier[] m_packageSuppliers;
127 		NativePath m_rootPath;
128 		SpecialDirs m_dirs;
129 		DubConfig m_config;
130 		NativePath m_projectPath;
131 		Project m_project;
132 		NativePath m_overrideSearchPath;
133 		string m_defaultCompiler;
134 		string m_defaultArchitecture;
135 		bool m_defaultLowMemory;
136 		string[string] m_defaultEnvironments;
137 		string[string] m_defaultBuildEnvironments;
138 		string[string] m_defaultRunEnvironments;
139 		string[string] m_defaultPreGenerateEnvironments;
140 		string[string] m_defaultPostGenerateEnvironments;
141 		string[string] m_defaultPreBuildEnvironments;
142 		string[string] m_defaultPostBuildEnvironments;
143 		string[string] m_defaultPreRunEnvironments;
144 		string[string] m_defaultPostRunEnvironments;
145 		
146 	}
147 
148 	/** The default placement location of fetched packages.
149 
150 		This property can be altered, so that packages which are downloaded as part
151 		of the normal upgrade process are stored in a certain location. This is
152 		how the "--local" and "--system" command line switches operate.
153 	*/
154 	PlacementLocation defaultPlacementLocation = PlacementLocation.user;
155 
156 
157 	/** Initializes the instance for use with a specific root package.
158 
159 		Note that a package still has to be loaded using one of the
160 		`loadPackage` overloads.
161 
162 		Params:
163 			root_path = Path to the root package
164 			additional_package_suppliers = A list of package suppliers to try
165 				before the suppliers found in the configurations files and the
166 				`defaultPackageSuppliers`.
167 			skip_registry = Can be used to skip using the configured package
168 				suppliers, as well as the default suppliers.
169 	*/
170 	this(string root_path = ".", PackageSupplier[] additional_package_suppliers = null,
171 			SkipPackageSuppliers skip_registry = SkipPackageSuppliers.none)
172 	{
173 		m_rootPath = NativePath(root_path);
174 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
175 
176 		init(m_rootPath);
177 
178 		m_packageSuppliers = getPackageSuppliers(additional_package_suppliers, skip_registry);
179 		m_packageManager = new PackageManager(m_rootPath, m_dirs.localRepository, m_dirs.systemSettings);
180 
181 		auto ccps = m_config.customCachePaths;
182 		if (ccps.length)
183 			m_packageManager.customCachePaths = ccps;
184 
185 		updatePackageSearchPath();
186 	}
187 
188 	unittest
189 	{
190 		scope (exit) environment.remove("DUB_REGISTRY");
191 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
192 		assert(dub.m_packageSuppliers.length == 0);
193 		environment["DUB_REGISTRY"] = "http://example.com/";
194 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
195 		assert(dub.m_packageSuppliers.length == 1);
196 		environment["DUB_REGISTRY"] = "http://example.com/;http://foo.com/";
197 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
198 		assert(dub.m_packageSuppliers.length == 2);
199 		dub = new Dub(".", [new RegistryPackageSupplier(URL("http://bar.com/"))], SkipPackageSuppliers.configured);
200 		assert(dub.m_packageSuppliers.length == 3);
201 	}
202 
203 	/** Get the list of package suppliers.
204 
205 		Params:
206 			additional_package_suppliers = A list of package suppliers to try
207 				before the suppliers found in the configurations files and the
208 				`defaultPackageSuppliers`.
209 			skip_registry = Can be used to skip using the configured package
210 				suppliers, as well as the default suppliers.
211 	*/
212 	public PackageSupplier[] getPackageSuppliers(PackageSupplier[] additional_package_suppliers, SkipPackageSuppliers skip_registry)
213 	{
214 		PackageSupplier[] ps = additional_package_suppliers;
215 
216 		if (skip_registry < SkipPackageSuppliers.all)
217 		{
218 			ps ~= environment.get("DUB_REGISTRY", null)
219 				.splitter(";")
220 				.map!(url => getRegistryPackageSupplier(url))
221 				.array;
222 		}
223 
224 		if (skip_registry < SkipPackageSuppliers.configured)
225 		{
226 			ps ~= m_config.registryURLs
227 				.map!(url => getRegistryPackageSupplier(url))
228 				.array;
229 		}
230 
231 		if (skip_registry < SkipPackageSuppliers.standard)
232 			ps ~= defaultPackageSuppliers();
233 
234 		return ps;
235 	}
236 
237 	/// ditto
238 	public PackageSupplier[] getPackageSuppliers(PackageSupplier[] additional_package_suppliers)
239 	{
240 		return getPackageSuppliers(additional_package_suppliers, m_config.skipRegistry);
241 	}
242 
243 	unittest
244 	{
245 		scope (exit) environment.remove("DUB_REGISTRY");
246 		auto dub = new Dub(".", null, SkipPackageSuppliers.none);
247 
248 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("none")]), null);
249 		assert(dub.getPackageSuppliers(null).length == 1);
250 
251 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("configured")]), null);
252 		assert(dub.getPackageSuppliers(null).length == 0);
253 
254 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("standard")]), null);
255 		assert(dub.getPackageSuppliers(null).length == 0);
256 
257 		environment["DUB_REGISTRY"] = "http://example.com/";
258 		assert(dub.getPackageSuppliers(null).length == 1);
259 	}
260 
261 	/** Initializes the instance with a single package search path, without
262 		loading a package.
263 
264 		This constructor corresponds to the "--bare" option of the command line
265 		interface. Use
266 	*/
267 	this(NativePath override_path)
268 	{
269 		init(NativePath());
270 		m_overrideSearchPath = override_path;
271 		m_packageManager = new PackageManager(override_path);
272 	}
273 
274 	private void init(NativePath root_path)
275 	{
276 		import std.file : tempDir;
277 		version(Windows) {
278 			m_dirs.systemSettings = NativePath(environment.get("ProgramData")) ~ "dub/";
279 			immutable appDataDir = environment.get("APPDATA");
280 			m_dirs.userSettings = NativePath(appDataDir) ~ "dub/";
281 			m_dirs.localRepository = NativePath(environment.get("LOCALAPPDATA", appDataDir)) ~ "dub";
282 		} else version(Posix){
283 			m_dirs.systemSettings = NativePath("/var/lib/dub/");
284 			m_dirs.userSettings = NativePath(environment.get("HOME")) ~ ".dub/";
285 			if (!m_dirs.userSettings.absolute)
286 				m_dirs.userSettings = NativePath(getcwd()) ~ m_dirs.userSettings;
287 			m_dirs.localRepository = m_dirs.userSettings;
288 		}
289 
290 		m_dirs.temp = NativePath(tempDir);
291 
292 		m_config = new DubConfig(jsonFromFile(m_dirs.systemSettings ~ "settings.json", true), m_config);
293 		m_config = new DubConfig(jsonFromFile(NativePath(thisExePath).parentPath ~ "../etc/dub/settings.json", true), m_config);
294 		m_config = new DubConfig(jsonFromFile(m_dirs.userSettings ~ "settings.json", true), m_config);
295 
296 		if (!root_path.empty)
297 			m_config = new DubConfig(jsonFromFile(root_path ~ "dub.settings.json", true), m_config);
298 
299 		determineDefaultCompiler();
300 
301 		m_defaultArchitecture = m_config.defaultArchitecture;
302 		m_defaultLowMemory = m_config.defaultLowMemory;
303 		m_defaultEnvironments = m_config.defaultEnvironments;
304 		m_defaultBuildEnvironments = m_config.defaultBuildEnvironments;
305 		m_defaultRunEnvironments = m_config.defaultRunEnvironments;
306 		m_defaultPreGenerateEnvironments = m_config.defaultPreGenerateEnvironments;
307 		m_defaultPostGenerateEnvironments = m_config.defaultPostGenerateEnvironments;
308 		m_defaultPreBuildEnvironments = m_config.defaultPreBuildEnvironments;
309 		m_defaultPostBuildEnvironments = m_config.defaultPostBuildEnvironments;
310 		m_defaultPreRunEnvironments = m_config.defaultPreRunEnvironments;
311 		m_defaultPostRunEnvironments = m_config.defaultPostRunEnvironments;
312 	}
313 
314 	@property void dryRun(bool v) { m_dryRun = v; }
315 
316 	/** Returns the root path (usually the current working directory).
317 	*/
318 	@property NativePath rootPath() const { return m_rootPath; }
319 	/// ditto
320 	@property void rootPath(NativePath root_path)
321 	{
322 		m_rootPath = root_path;
323 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
324 	}
325 
326 	/// Returns the name listed in the dub.json of the current
327 	/// application.
328 	@property string projectName() const { return m_project.name; }
329 
330 	@property NativePath projectPath() const { return m_projectPath; }
331 
332 	@property string[] configurations() const { return m_project.configurations; }
333 
334 	@property inout(PackageManager) packageManager() inout { return m_packageManager; }
335 
336 	@property inout(Project) project() inout { return m_project; }
337 
338 	/** Returns the default compiler binary to use for building D code.
339 
340 		If set, the "defaultCompiler" field of the DUB user or system
341 		configuration file will be used. Otherwise the PATH environment variable
342 		will be searched for files named "dmd", "gdc", "gdmd", "ldc2", "ldmd2"
343 		(in that order, taking into account operating system specific file
344 		extensions) and the first match is returned. If no match is found, "dmd"
345 		will be used.
346 	*/
347 	@property string defaultCompiler() const { return m_defaultCompiler; }
348 
349 	/** Returns the default architecture to use for building D code.
350 
351 		If set, the "defaultArchitecture" field of the DUB user or system
352 		configuration file will be used. Otherwise null will be returned.
353 	*/
354 	@property string defaultArchitecture() const { return m_defaultArchitecture; }
355 
356 	/** Returns the default low memory option to use for building D code.
357 
358 		If set, the "defaultLowMemory" field of the DUB user or system
359 		configuration file will be used. Otherwise false will be returned.
360 	*/
361 	@property bool defaultLowMemory() const { return m_defaultLowMemory; }
362 	
363 	@property const(string[string]) defaultEnvironments() const { return m_defaultEnvironments; }
364 	@property const(string[string]) defaultBuildEnvironments() const { return m_defaultBuildEnvironments; }
365 	@property const(string[string]) defaultRunEnvironments() const { return m_defaultRunEnvironments; }
366 	@property const(string[string]) defaultPreGenerateEnvironments() const { return m_defaultPreGenerateEnvironments; }
367 	@property const(string[string]) defaultPostGenerateEnvironments() const { return m_defaultPostGenerateEnvironments; }
368 	@property const(string[string]) defaultPreBuildEnvironments() const { return m_defaultPreBuildEnvironments; }
369 	@property const(string[string]) defaultPostBuildEnvironments() const { return m_defaultPostBuildEnvironments; }
370 	@property const(string[string]) defaultPreRunEnvironments() const { return m_defaultPreRunEnvironments; }
371 	@property const(string[string]) defaultPostRunEnvironments() const { return m_defaultPostRunEnvironments; }
372 
373 	/** Loads the package that resides within the configured `rootPath`.
374 	*/
375 	void loadPackage()
376 	{
377 		loadPackage(m_rootPath);
378 	}
379 
380 	/// Loads the package from the specified path as the main project package.
381 	void loadPackage(NativePath path)
382 	{
383 		m_projectPath = path;
384 		updatePackageSearchPath();
385 		m_project = new Project(m_packageManager, m_projectPath);
386 	}
387 
388 	/// Loads a specific package as the main project package (can be a sub package)
389 	void loadPackage(Package pack)
390 	{
391 		m_projectPath = pack.path;
392 		updatePackageSearchPath();
393 		m_project = new Project(m_packageManager, pack);
394 	}
395 
396 	/** Loads a single file package.
397 
398 		Single-file packages are D files that contain a package receipe comment
399 		at their top. A recipe comment must be a nested `/+ ... +/` style
400 		comment, containing the virtual recipe file name and a colon, followed by the
401 		recipe contents (what would normally be in dub.sdl/dub.json).
402 
403 		Example:
404 		---
405 		/+ dub.sdl:
406 		   name "test"
407 		   dependency "vibe-d" version="~>0.7.29"
408 		+/
409 		import vibe.http.server;
410 
411 		void main()
412 		{
413 			auto settings = new HTTPServerSettings;
414 			settings.port = 8080;
415 			listenHTTP(settings, &hello);
416 		}
417 
418 		void hello(HTTPServerRequest req, HTTPServerResponse res)
419 		{
420 			res.writeBody("Hello, World!");
421 		}
422 		---
423 
424 		The script above can be invoked with "dub --single test.d".
425 	*/
426 	void loadSingleFilePackage(NativePath path)
427 	{
428 		import dub.recipe.io : parsePackageRecipe;
429 		import std.file : mkdirRecurse, readText;
430 		import std.path : baseName, stripExtension;
431 
432 		path = makeAbsolute(path);
433 
434 		string file_content = readText(path.toNativeString());
435 
436 		if (file_content.startsWith("#!")) {
437 			auto idx = file_content.indexOf('\n');
438 			enforce(idx > 0, "The source fine doesn't contain anything but a shebang line.");
439 			file_content = file_content[idx+1 .. $];
440 		}
441 
442 		file_content = file_content.strip();
443 
444 		string recipe_content;
445 
446 		if (file_content.startsWith("/+")) {
447 			file_content = file_content[2 .. $];
448 			auto idx = file_content.indexOf("+/");
449 			enforce(idx >= 0, "Missing \"+/\" to close comment.");
450 			recipe_content = file_content[0 .. idx].strip();
451 		} else throw new Exception("The source file must start with a recipe comment.");
452 
453 		auto nidx = recipe_content.indexOf('\n');
454 
455 		auto idx = recipe_content.indexOf(':');
456 		enforce(idx > 0 && (nidx < 0 || nidx > idx),
457 			"The first line of the recipe comment must list the recipe file name followed by a colon (e.g. \"/+ dub.sdl:\").");
458 		auto recipe_filename = recipe_content[0 .. idx];
459 		recipe_content = recipe_content[idx+1 .. $];
460 		auto recipe_default_package_name = path.toString.baseName.stripExtension.strip;
461 
462 		auto recipe = parsePackageRecipe(recipe_content, recipe_filename, null, recipe_default_package_name);
463 		import dub.internal.vibecompat.core.log; logInfo("parsePackageRecipe %s", recipe_filename);
464 		enforce(recipe.buildSettings.sourceFiles.length == 0, "Single-file packages are not allowed to specify source files.");
465 		enforce(recipe.buildSettings.sourcePaths.length == 0, "Single-file packages are not allowed to specify source paths.");
466 		enforce(recipe.buildSettings.importPaths.length == 0, "Single-file packages are not allowed to specify import paths.");
467 		recipe.buildSettings.sourceFiles[""] = [path.toNativeString()];
468 		recipe.buildSettings.sourcePaths[""] = [];
469 		recipe.buildSettings.importPaths[""] = [];
470 		recipe.buildSettings.mainSourceFile = path.toNativeString();
471 		if (recipe.buildSettings.targetType == TargetType.autodetect)
472 			recipe.buildSettings.targetType = TargetType.executable;
473 
474 		auto pack = new Package(recipe, path.parentPath, null, "~master");
475 		loadPackage(pack);
476 	}
477 	/// ditto
478 	void loadSingleFilePackage(string path)
479 	{
480 		loadSingleFilePackage(NativePath(path));
481 	}
482 
483 	deprecated("Instantiate a Dub instance with the single-argument constructor: `new Dub(path)`")
484 	void overrideSearchPath(NativePath path)
485 	{
486 		if (!path.absolute) path = NativePath(getcwd()) ~ path;
487 		m_overrideSearchPath = path;
488 		updatePackageSearchPath();
489 	}
490 
491 	/** Gets the default configuration for a particular build platform.
492 
493 		This forwards to `Project.getDefaultConfiguration` and requires a
494 		project to be loaded.
495 	*/
496 	string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); }
497 
498 	/** Attempts to upgrade the dependency selection of the loaded project.
499 
500 		Params:
501 			options = Flags that control how the upgrade is carried out
502 			packages_to_upgrade = Optional list of packages. If this list
503 				contains one or more packages, only those packages will
504 				be upgraded. Otherwise, all packages will be upgraded at
505 				once.
506 	*/
507 	void upgrade(UpgradeOptions options, string[] packages_to_upgrade = null)
508 	{
509 		// clear non-existent version selections
510 		if (!(options & UpgradeOptions.upgrade)) {
511 			next_pack:
512 			foreach (p; m_project.selections.selectedPackages) {
513 				auto dep = m_project.selections.getSelectedVersion(p);
514 				if (!dep.path.empty) {
515 					auto path = dep.path;
516 					if (!path.absolute) path = this.rootPath ~ path;
517 					try if (m_packageManager.getOrLoadPackage(path)) continue;
518 					catch (Exception e) { logDebug("Failed to load path based selection: %s", e.toString().sanitize); }
519 				} else if (!dep.repository.empty) {
520 					if (m_packageManager.loadSCMPackage(getBasePackageName(p), dep))
521 						continue;
522 				} else {
523 					if (m_packageManager.getPackage(p, dep.version_)) continue;
524 					foreach (ps; m_packageSuppliers) {
525 						try {
526 							auto versions = ps.getVersions(p);
527 							if (versions.canFind!(v => dep.matches(v)))
528 								continue next_pack;
529 						} catch (Exception e) {
530 							logWarn("Error querying versions for %s, %s: %s", p, ps.description, e.msg);
531 							logDebug("Full error: %s", e.toString().sanitize());
532 						}
533 					}
534 				}
535 
536 				logWarn("Selected package %s %s doesn't exist. Using latest matching version instead.", p, dep);
537 				m_project.selections.deselectVersion(p);
538 			}
539 		}
540 
541 		Dependency[string] versions;
542 		auto resolver = new DependencyVersionResolver(this, options);
543 		foreach (p; packages_to_upgrade)
544 			resolver.addPackageToUpgrade(p);
545 		versions = resolver.resolve(m_project.rootPackage, m_project.selections);
546 
547 		if (options & UpgradeOptions.dryRun) {
548 			bool any = false;
549 			string rootbasename = getBasePackageName(m_project.rootPackage.name);
550 
551 			foreach (p, ver; versions) {
552 				if (!ver.path.empty || !ver.repository.empty) continue;
553 
554 				auto basename = getBasePackageName(p);
555 				if (basename == rootbasename) continue;
556 
557 				if (!m_project.selections.hasSelectedVersion(basename)) {
558 					logInfo("Package %s would be selected with version %s.",
559 						basename, ver);
560 					any = true;
561 					continue;
562 				}
563 				auto sver = m_project.selections.getSelectedVersion(basename);
564 				if (!sver.path.empty || !sver.repository.empty) continue;
565 				if (ver.version_ <= sver.version_) continue;
566 				logInfo("Package %s would be upgraded from %s to %s.",
567 					basename, sver, ver);
568 				any = true;
569 			}
570 			if (any) logInfo("Use \"dub upgrade\" to perform those changes.");
571 			return;
572 		}
573 
574 		foreach (p; versions.byKey) {
575 			auto ver = versions[p]; // Workaround for DMD 2.070.0 AA issue (crashes in aaApply2 if iterating by key+value)
576 			assert(!p.canFind(":"), "Resolved packages contain a sub package!?: "~p);
577 			Package pack;
578 			if (!ver.path.empty) {
579 				try pack = m_packageManager.getOrLoadPackage(ver.path);
580 				catch (Exception e) {
581 					logDebug("Failed to load path based selection: %s", e.toString().sanitize);
582 					continue;
583 				}
584 			} else if (!ver.repository.empty) {
585 				pack = m_packageManager.loadSCMPackage(p, ver);
586 			} else {
587 				pack = m_packageManager.getBestPackage(p, ver);
588 				if (pack && m_packageManager.isManagedPackage(pack)
589 					&& ver.version_.isBranch && (options & UpgradeOptions.upgrade) != 0)
590 				{
591 					// TODO: only re-install if there is actually a new commit available
592 					logInfo("Re-installing branch based dependency %s %s", p, ver.toString());
593 					m_packageManager.remove(pack);
594 					pack = null;
595 				}
596 			}
597 
598 			FetchOptions fetchOpts;
599 			fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none;
600 			if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version");
601 			if ((options & UpgradeOptions.select) && p != m_project.rootPackage.name) {
602 				if (!ver.repository.empty) {
603 					m_project.selections.selectVersionWithRepository(p, ver.repository, ver.versionSpec);
604 				} else if (ver.path.empty) {
605 					m_project.selections.selectVersion(p, ver.version_);
606 				} else {
607 					NativePath relpath = ver.path;
608 					if (relpath.absolute) relpath = relpath.relativeTo(m_project.rootPackage.path);
609 					m_project.selections.selectVersion(p, relpath);
610 				}
611 			}
612 		}
613 
614 		string[] missingDependenciesBeforeReinit = m_project.missingDependencies;
615 		m_project.reinit();
616 
617 		if (!m_project.hasAllDependencies) {
618 			auto resolvedDependencies = setDifference(
619 					assumeSorted(missingDependenciesBeforeReinit),
620 					assumeSorted(m_project.missingDependencies)
621 				);
622 			if (!resolvedDependencies.empty)
623 				upgrade(options, m_project.missingDependencies);
624 		}
625 
626 		if ((options & UpgradeOptions.select) && !(options & (UpgradeOptions.noSaveSelections | UpgradeOptions.dryRun)))
627 			m_project.saveSelections();
628 	}
629 
630 	/** Generate project files for a specified generator.
631 
632 		Any existing project files will be overridden.
633 	*/
634 	void generateProject(string ide, GeneratorSettings settings)
635 	{
636 		auto generator = createProjectGenerator(ide, m_project);
637 		if (m_dryRun) return; // TODO: pass m_dryRun to the generator
638 		generator.generate(settings);
639 	}
640 
641 	/** Executes tests on the current project.
642 
643 		Throws an exception, if unittests failed.
644 	*/
645 	void testProject(GeneratorSettings settings, string config, NativePath custom_main_file)
646 	{
647 		if (!custom_main_file.empty && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file;
648 
649 		if (config.length == 0) {
650 			// if a custom main file was given, favor the first library configuration, so that it can be applied
651 			if (!custom_main_file.empty) config = m_project.getDefaultConfiguration(settings.platform, false);
652 			// else look for a "unittest" configuration
653 			if (!config.length && m_project.rootPackage.configurations.canFind("unittest")) config = "unittest";
654 			// if not found, fall back to the first "library" configuration
655 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, false);
656 			// if still nothing found, use the first executable configuration
657 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, true);
658 		}
659 
660 		auto generator = createProjectGenerator("build", m_project);
661 
662 		auto test_config = format("%s-test-%s", m_project.rootPackage.name.replace(".", "-").replace(":", "-"), config);
663 
664 		BuildSettings lbuildsettings = settings.buildSettings;
665 		m_project.addBuildSettings(lbuildsettings, settings, config, null, true);
666 		if (lbuildsettings.targetType == TargetType.none) {
667 			logInfo(`Configuration '%s' has target type "none". Skipping test.`, config);
668 			return;
669 		}
670 
671 		if (lbuildsettings.targetType == TargetType.executable && config == "unittest") {
672 			logInfo("Running custom 'unittest' configuration.", config);
673 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
674 			settings.config = config;
675 		} else if (lbuildsettings.sourceFiles.empty) {
676 			logInfo(`No source files found in configuration '%s'. Falling back to "dub -b unittest".`, config);
677 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
678 			settings.config = m_project.getDefaultConfiguration(settings.platform);
679 		} else {
680 			import std.algorithm : remove;
681 
682 			logInfo(`Generating test runner configuration '%s' for '%s' (%s).`, test_config, config, lbuildsettings.targetType);
683 
684 			BuildSettingsTemplate tcinfo = m_project.rootPackage.recipe.getConfiguration(config).buildSettings;
685 			tcinfo.targetType = TargetType.executable;
686 			tcinfo.targetName = test_config;
687 
688 			auto mainfil = tcinfo.mainSourceFile;
689 			if (!mainfil.length) mainfil = m_project.rootPackage.recipe.buildSettings.mainSourceFile;
690 
691 			string custommodname;
692 			if (!custom_main_file.empty) {
693 				import std.path;
694 				tcinfo.sourceFiles[""] ~= custom_main_file.relativeTo(m_project.rootPackage.path).toNativeString();
695 				tcinfo.importPaths[""] ~= custom_main_file.parentPath.toNativeString();
696 				custommodname = custom_main_file.head.name.baseName(".d");
697 			}
698 
699 			// prepare the list of tested modules
700 
701 			string[] import_modules;
702 			if (settings.single)
703 				lbuildsettings.importPaths ~= NativePath(mainfil).parentPath.toNativeString;
704 			bool firstTimePackage = true;
705 			foreach (file; lbuildsettings.sourceFiles) {
706 				if (file.endsWith(".d")) {
707 					auto fname = NativePath(file).head.name;
708 					NativePath msf = NativePath(mainfil);
709 					if (msf.absolute)
710 						msf = msf.relativeTo(m_project.rootPackage.path);
711 					if (!settings.single && NativePath(file).relativeTo(m_project.rootPackage.path) == msf) {
712 						logWarn("Excluding main source file %s from test.", mainfil);
713 						tcinfo.excludedSourceFiles[""] ~= mainfil;
714 						continue;
715 					}
716 					if (fname == "package.d") {
717 						if (firstTimePackage) {
718 							firstTimePackage = false;
719 							logDiagnostic("Excluding package.d file from test due to https://issues.dlang.org/show_bug.cgi?id=11847");
720 						}
721 						continue;
722 					}
723 					import_modules ~= dub.internal.utils.determineModuleName(lbuildsettings, NativePath(file), m_project.rootPackage.path);
724 				}
725 			}
726 
727 			NativePath mainfile;
728 			if (settings.tempBuild)
729 				mainfile = getTempFile("dub_test_root", ".d");
730 			else {
731 				import dub.generators.build : computeBuildName;
732 				mainfile = m_project.rootPackage.path ~ format(".dub/code/%s_dub_test_root.d", computeBuildName(test_config, settings, import_modules));
733 			}
734 
735 			mkdirRecurse(mainfile.parentPath.toNativeString());
736 
737 			bool regenerateMainFile = settings.force || !existsFile(mainfile);
738 			auto escapedMainFile = mainfile.toNativeString().replace("$", "$$");
739 			// generate main file
740 			tcinfo.sourceFiles[""] ~= escapedMainFile;
741 			tcinfo.mainSourceFile = escapedMainFile;
742 
743 			if (!m_dryRun && regenerateMainFile) {
744 				auto fil = openFile(mainfile, FileMode.createTrunc);
745 				scope(exit) fil.close();
746 				fil.write("module dub_test_root;\n");
747 				fil.write("import std.typetuple;\n");
748 				foreach (mod; import_modules) fil.write(format("static import %s;\n", mod));
749 				fil.write("alias allModules = TypeTuple!(");
750 				foreach (i, mod; import_modules) {
751 					if (i > 0) fil.write(", ");
752 					fil.write(mod);
753 				}
754 				fil.write(");\n");
755 				if (custommodname.length) {
756 					fil.write(format("import %s;\n", custommodname));
757 				} else {
758 					fil.write(q{
759 						import std.stdio;
760 						import core.runtime;
761 
762 						void main() { writeln("All unit tests have been run successfully."); }
763 						shared static this() {
764 							version (Have_tested) {
765 								import tested;
766 								import core.runtime;
767 								import std.exception;
768 								Runtime.moduleUnitTester = () => true;
769 								enforce(runUnitTests!allModules(new ConsoleTestResultWriter), "Unit tests failed.");
770 							}
771 						}
772 					});
773 				}
774 			}
775 			m_project.rootPackage.recipe.configurations ~= ConfigurationInfo(test_config, tcinfo);
776 			m_project = new Project(m_packageManager, m_project.rootPackage);
777 
778 			settings.config = test_config;
779 		}
780 
781 		generator.generate(settings);
782 	}
783 
784 	/** Executes D-Scanner tests on the current project. **/
785 	void lintProject(string[] args)
786 	{
787 		import std.path : buildPath, buildNormalizedPath;
788 
789 		if (m_dryRun) return;
790 
791 		auto tool = "dscanner";
792 
793 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
794 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
795 		if (!tool_pack) {
796 			logInfo("%s is not present, getting and storing it user wide", tool);
797 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
798 		}
799 
800 		auto dscanner_dub = new Dub(null, m_packageSuppliers);
801 		dscanner_dub.loadPackage(tool_pack.path);
802 		dscanner_dub.upgrade(UpgradeOptions.select);
803 
804 		auto compiler_binary = this.defaultCompiler;
805 
806 		GeneratorSettings settings;
807 		settings.config = "application";
808 		settings.compiler = getCompiler(compiler_binary);
809 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
810 		settings.buildType = "debug";
811 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
812 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
813 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
814 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
815 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
816 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
817 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
818 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
819 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
820 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
821 		settings.run = true;
822 
823 		foreach (dependencyPackage; m_project.dependencies)
824 		{
825 			auto cfgs = m_project.getPackageConfigs(settings.platform, null, true);
826 			auto buildSettings = dependencyPackage.getBuildSettings(settings.platform, cfgs[dependencyPackage.name]);
827 			foreach (importPath; buildSettings.importPaths) {
828 				settings.runArgs ~= ["-I", buildNormalizedPath(dependencyPackage.path.toNativeString(), importPath.idup)];
829 			}
830 		}
831 
832 		string configFilePath = buildPath(m_project.rootPackage.path.toNativeString(), "dscanner.ini");
833 		if (!args.canFind("--config") && exists(configFilePath)) {
834 			settings.runArgs ~= ["--config", configFilePath];
835 		}
836 
837 		settings.runArgs ~= args ~ [m_project.rootPackage.path.toNativeString()];
838 		dscanner_dub.generateProject("build", settings);
839 	}
840 
841 	/** Prints the specified build settings necessary for building the root package.
842 	*/
843 	void listProjectData(GeneratorSettings settings, string[] requestedData, ListBuildSettingsFormat list_type)
844 	{
845 		import std.stdio;
846 		import std.ascii : newline;
847 
848 		// Split comma-separated lists
849 		string[] requestedDataSplit =
850 			requestedData
851 			.map!(a => a.splitter(",").map!strip)
852 			.joiner()
853 			.array();
854 
855 		auto data = m_project.listBuildSettings(settings, requestedDataSplit, list_type);
856 
857 		string delimiter;
858 		final switch (list_type) with (ListBuildSettingsFormat) {
859 			case list: delimiter = newline ~ newline; break;
860 			case listNul: delimiter = "\0\0"; break;
861 			case commandLine: delimiter = " "; break;
862 			case commandLineNul: delimiter = "\0\0"; break;
863 		}
864 
865 		write(data.joiner(delimiter));
866 		if (delimiter != "\0\0") writeln();
867 	}
868 
869 	/// Cleans intermediate/cache files of the given package
870 	void cleanPackage(NativePath path)
871 	{
872 		logInfo("Cleaning package at %s...", path.toNativeString());
873 		enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString());
874 
875 		// TODO: clear target files and copy files
876 
877 		if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString());
878 		if (existsFile(path ~ ".dub/metadata_cache.json")) std.file.remove((path ~ ".dub/metadata_cache.json").toNativeString());
879 
880 		auto p = Package.load(path);
881 		if (p.getBuildSettings().targetType == TargetType.none) {
882 			foreach (sp; p.subPackages.filter!(sp => !sp.path.empty)) {
883 				cleanPackage(path ~ sp.path);
884 			}
885 		}
886 	}
887 
888 	/// Fetches the package matching the dependency and places it in the specified location.
889 	Package fetch(string packageId, const Dependency dep, PlacementLocation location, FetchOptions options, string reason = "")
890 	{
891 		auto basePackageName = getBasePackageName(packageId);
892 		Json pinfo;
893 		PackageSupplier supplier;
894 		foreach(ps; m_packageSuppliers){
895 			try {
896 				pinfo = ps.fetchPackageRecipe(basePackageName, dep, (options & FetchOptions.usePrerelease) != 0);
897 				if (pinfo.type == Json.Type.null_)
898 					continue;
899 				supplier = ps;
900 				break;
901 			} catch(Exception e) {
902 				logWarn("Package %s not found for %s: %s", packageId, ps.description, e.msg);
903 				logDebug("Full error: %s", e.toString().sanitize());
904 			}
905 		}
906 		enforce(pinfo.type != Json.Type.undefined, "No package "~packageId~" was found matching the dependency "~dep.toString());
907 		string ver = pinfo["version"].get!string;
908 
909 		NativePath placement;
910 		final switch (location) {
911 			case PlacementLocation.local: placement = m_rootPath ~ ".dub/packages/"; break;
912 			case PlacementLocation.user: placement = m_dirs.localRepository ~ "packages/"; break;
913 			case PlacementLocation.system: placement = m_dirs.systemSettings ~ "packages/"; break;
914 		}
915 
916 		// always upgrade branch based versions - TODO: actually check if there is a new commit available
917 		Package existing;
918 		try existing = m_packageManager.getPackage(packageId, ver, placement);
919 		catch (Exception e) {
920 			logWarn("Failed to load existing package %s: %s", ver, e.msg);
921 			logDiagnostic("Full error: %s", e.toString().sanitize);
922 		}
923 
924 		if (options & FetchOptions.printOnly) {
925 			if (existing && existing.version_ != Version(ver))
926 				logInfo("A new version for %s is available (%s -> %s). Run \"dub upgrade %s\" to switch.",
927 					packageId, existing.version_, ver, packageId);
928 			return null;
929 		}
930 
931 		if (existing) {
932 			if (!ver.startsWith("~") || !(options & FetchOptions.forceBranchUpgrade) || location == PlacementLocation.local) {
933 				// TODO: support git working trees by performing a "git pull" instead of this
934 				logDiagnostic("Package %s %s (%s) is already present with the latest version, skipping upgrade.",
935 					packageId, ver, placement);
936 				return existing;
937 			} else {
938 				logInfo("Removing %s %s to prepare replacement with a new version.", packageId, ver);
939 				if (!m_dryRun) m_packageManager.remove(existing);
940 			}
941 		}
942 
943 		if (reason.length) logInfo("Fetching %s %s (%s)...", packageId, ver, reason);
944 		else logInfo("Fetching %s %s...", packageId, ver);
945 		if (m_dryRun) return null;
946 
947 		logDebug("Acquiring package zip file");
948 
949 		auto clean_package_version = ver[ver.startsWith("~") ? 1 : 0 .. $];
950 		clean_package_version = clean_package_version.replace("+", "_"); // + has special meaning for Optlink
951 		if (!placement.existsFile())
952 			mkdirRecurse(placement.toNativeString());
953 		NativePath dstpath = placement ~ (basePackageName ~ "-" ~ clean_package_version);
954 		if (!dstpath.existsFile())
955 			mkdirRecurse(dstpath.toNativeString());
956 
957 		// Support libraries typically used with git submodules like ae.
958 		// Such libraries need to have ".." as import path but this can create
959 		// import path leakage.
960 		dstpath = dstpath ~ basePackageName;
961 
962 		import std.datetime : seconds;
963 		auto lock = lockFile(dstpath.toNativeString() ~ ".lock", 30.seconds); // possibly wait for other dub instance
964 		if (dstpath.existsFile())
965 		{
966 			m_packageManager.refresh(false);
967 			return m_packageManager.getPackage(packageId, ver, dstpath);
968 		}
969 
970 		// repeat download on corrupted zips, see #1336
971 		foreach_reverse (i; 0..3)
972 		{
973 			import std.zip : ZipException;
974 
975 			auto path = getTempFile(basePackageName, ".zip");
976 			supplier.fetchPackage(path, basePackageName, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
977 			scope(exit) std.file.remove(path.toNativeString());
978 			logDiagnostic("Placing to %s...", placement.toNativeString());
979 
980 			try {
981 				m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
982 				return m_packageManager.getPackage(packageId, ver, dstpath);
983 			} catch (ZipException e) {
984 				logInfo("Failed to extract zip archive for %s %s...", packageId, ver);
985 				// rethrow the exception at the end of the loop
986 				if (i == 0)
987 					throw e;
988 			}
989 		}
990 		assert(0, "Should throw a ZipException instead.");
991 	}
992 
993 	/** Removes a specific locally cached package.
994 
995 		This will delete the package files from disk and removes the
996 		corresponding entry from the list of known packages.
997 
998 		Params:
999 			pack = Package instance to remove
1000 	*/
1001 	void remove(in Package pack)
1002 	{
1003 		logInfo("Removing %s in %s", pack.name, pack.path.toNativeString());
1004 		if (!m_dryRun) m_packageManager.remove(pack);
1005 	}
1006 
1007 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1008 	void remove(in Package pack, bool force_remove)
1009 	{
1010 		remove(pack);
1011 	}
1012 
1013 	/// @see remove(string, string, RemoveLocation)
1014 	enum RemoveVersionWildcard = "*";
1015 
1016 	/** Removes one or more versions of a locally cached package.
1017 
1018 		This will remove a given package with a specified version from the
1019 		given location. It will remove at most one package, unless `version_`
1020 		is set to `RemoveVersionWildcard`.
1021 
1022 		Params:
1023 			package_id = Name of the package to be removed
1024 			location_ = Specifies the location to look for the given package
1025 				name/version.
1026 			resolve_version = Callback to select package version.
1027 	*/
1028 	void remove(string package_id, PlacementLocation location,
1029 				scope size_t delegate(in Package[] packages) resolve_version)
1030 	{
1031 		enforce(!package_id.empty);
1032 		if (location == PlacementLocation.local) {
1033 			logInfo("To remove a locally placed package, make sure you don't have any data"
1034 					~ "\nleft in it's directory and then simply remove the whole directory.");
1035 			throw new Exception("dub cannot remove locally installed packages.");
1036 		}
1037 
1038 		Package[] packages;
1039 
1040 		// Retrieve packages to be removed.
1041 		foreach(pack; m_packageManager.getPackageIterator(package_id))
1042 			if (m_packageManager.isManagedPackage(pack))
1043 				packages ~= pack;
1044 
1045 		// Check validity of packages to be removed.
1046 		if(packages.empty) {
1047 			throw new Exception("Cannot find package to remove. ("
1048 				~ "id: '" ~ package_id ~ "', location: '" ~ to!string(location) ~ "'"
1049 				~ ")");
1050 		}
1051 
1052 		// Sort package list in ascending version order
1053 		packages.sort!((a, b) => a.version_ < b.version_);
1054 
1055 		immutable idx = resolve_version(packages);
1056 		if (idx == size_t.max)
1057 			return;
1058 		else if (idx != packages.length)
1059 			packages = packages[idx .. idx + 1];
1060 
1061 		logDebug("Removing %s packages.", packages.length);
1062 		foreach(pack; packages) {
1063 			try {
1064 				remove(pack);
1065 				logInfo("Removed %s, version %s.", package_id, pack.version_);
1066 			} catch (Exception e) {
1067 				logError("Failed to remove %s %s: %s", package_id, pack.version_, e.msg);
1068 				logInfo("Continuing with other packages (if any).");
1069 			}
1070 		}
1071 	}
1072 
1073 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1074 	void remove(string package_id, PlacementLocation location, bool force_remove,
1075 				scope size_t delegate(in Package[] packages) resolve_version)
1076 	{
1077 		remove(package_id, location, resolve_version);
1078 	}
1079 
1080 	/** Removes a specific version of a package.
1081 
1082 		Params:
1083 			package_id = Name of the package to be removed
1084 			version_ = Identifying a version or a wild card. If an empty string
1085 				is passed, the package will be removed from the location, if
1086 				there is only one version retrieved. This will throw an
1087 				exception, if there are multiple versions retrieved.
1088 			location_ = Specifies the location to look for the given package
1089 				name/version.
1090 	 */
1091 	void remove(string package_id, string version_, PlacementLocation location)
1092 	{
1093 		remove(package_id, location, (in packages) {
1094 			if (version_ == RemoveVersionWildcard || version_.empty)
1095 				return packages.length;
1096 
1097 			foreach (i, p; packages) {
1098 				if (p.version_ == Version(version_))
1099 					return i;
1100 			}
1101 			throw new Exception("Cannot find package to remove. ("
1102 				~ "id: '" ~ package_id ~ "', version: '" ~ version_ ~ "', location: '" ~ to!string(location) ~ "'"
1103 				~ ")");
1104 		});
1105 	}
1106 
1107 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1108 	void remove(string package_id, string version_, PlacementLocation location, bool force_remove)
1109 	{
1110 		remove(package_id, version_, location);
1111 	}
1112 
1113 	/** Adds a directory to the list of locally known packages.
1114 
1115 		Forwards to `PackageManager.addLocalPackage`.
1116 
1117 		Params:
1118 			path = Path to the package
1119 			ver = Optional version to associate with the package (can be left
1120 				empty)
1121 			system = Make the package known system wide instead of user wide
1122 				(requires administrator privileges).
1123 
1124 		See_Also: `removeLocalPackage`
1125 	*/
1126 	void addLocalPackage(string path, string ver, bool system)
1127 	{
1128 		if (m_dryRun) return;
1129 		m_packageManager.addLocalPackage(makeAbsolute(path), ver, system ? LocalPackageType.system : LocalPackageType.user);
1130 	}
1131 
1132 	/** Removes a directory from the list of locally known packages.
1133 
1134 		Forwards to `PackageManager.removeLocalPackage`.
1135 
1136 		Params:
1137 			path = Path to the package
1138 			system = Make the package known system wide instead of user wide
1139 				(requires administrator privileges).
1140 
1141 		See_Also: `addLocalPackage`
1142 	*/
1143 	void removeLocalPackage(string path, bool system)
1144 	{
1145 		if (m_dryRun) return;
1146 		m_packageManager.removeLocalPackage(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1147 	}
1148 
1149 	/** Registers a local directory to search for packages to use for satisfying
1150 		dependencies.
1151 
1152 		Params:
1153 			path = Path to a directory containing package directories
1154 			system = Make the package known system wide instead of user wide
1155 				(requires administrator privileges).
1156 
1157 		See_Also: `removeSearchPath`
1158 	*/
1159 	void addSearchPath(string path, bool system)
1160 	{
1161 		if (m_dryRun) return;
1162 		m_packageManager.addSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1163 	}
1164 
1165 	/** Unregisters a local directory search path.
1166 
1167 		Params:
1168 			path = Path to a directory containing package directories
1169 			system = Make the package known system wide instead of user wide
1170 				(requires administrator privileges).
1171 
1172 		See_Also: `addSearchPath`
1173 	*/
1174 	void removeSearchPath(string path, bool system)
1175 	{
1176 		if (m_dryRun) return;
1177 		m_packageManager.removeSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1178 	}
1179 
1180 	/** Queries all package suppliers with the given query string.
1181 
1182 		Returns a list of tuples, where the first entry is the human readable
1183 		name of the package supplier and the second entry is the list of
1184 		matched packages.
1185 
1186 		See_Also: `PackageSupplier.searchPackages`
1187 	*/
1188 	auto searchPackages(string query)
1189 	{
1190 		import std.typecons : Tuple, tuple;
1191 		Tuple!(string, PackageSupplier.SearchResult[])[] results;
1192 		foreach (ps; this.m_packageSuppliers) {
1193 			try
1194 				results ~= tuple(ps.description, ps.searchPackages(query));
1195 			catch (Exception e) {
1196 				logWarn("Searching %s for '%s' failed: %s", ps.description, query, e.msg);
1197 			}
1198 		}
1199 		return results.filter!(tup => tup[1].length);
1200 	}
1201 
1202 	/** Returns a list of all available versions (including branches) for a
1203 		particular package.
1204 
1205 		The list returned is based on the registered package suppliers. Local
1206 		packages are not queried in the search for versions.
1207 
1208 		See_also: `getLatestVersion`
1209 	*/
1210 	Version[] listPackageVersions(string name)
1211 	{
1212 		Version[] versions;
1213 		auto basePackageName = getBasePackageName(name);
1214 		foreach (ps; this.m_packageSuppliers) {
1215 			try versions ~= ps.getVersions(basePackageName);
1216 			catch (Exception e) {
1217 				logWarn("Failed to get versions for package %s on provider %s: %s", name, ps.description, e.msg);
1218 			}
1219 		}
1220 		return versions.sort().uniq.array;
1221 	}
1222 
1223 	/** Returns the latest available version for a particular package.
1224 
1225 		This function returns the latest numbered version of a package. If no
1226 		numbered versions are available, it will return an available branch,
1227 		preferring "~master".
1228 
1229 		Params:
1230 			package_name: The name of the package in question.
1231 			prefer_stable: If set to `true` (the default), returns the latest
1232 				stable version, even if there are newer pre-release versions.
1233 
1234 		See_also: `listPackageVersions`
1235 	*/
1236 	Version getLatestVersion(string package_name, bool prefer_stable = true)
1237 	{
1238 		auto vers = listPackageVersions(package_name);
1239 		enforce(!vers.empty, "Failed to find any valid versions for a package name of '"~package_name~"'.");
1240 		auto final_versions = vers.filter!(v => !v.isBranch && !v.isPreRelease).array;
1241 		if (prefer_stable && final_versions.length) return final_versions[$-1];
1242 		else return vers[$-1];
1243 	}
1244 
1245 	/** Initializes a directory with a package skeleton.
1246 
1247 		Params:
1248 			path = Path of the directory to create the new package in. The
1249 				directory will be created if it doesn't exist.
1250 			deps = List of dependencies to add to the package recipe.
1251 			type = Specifies the type of the application skeleton to use.
1252 			format = Determines the package recipe format to use.
1253 			recipe_callback = Optional callback that can be used to
1254 				customize the recipe before it gets written.
1255 	*/
1256 	void createEmptyPackage(NativePath path, string[] deps, string type,
1257 		PackageFormat format = PackageFormat.sdl,
1258 		scope void delegate(ref PackageRecipe, ref PackageFormat) recipe_callback = null,
1259 		string[] app_args = [])
1260 	{
1261 		if (!path.absolute) path = m_rootPath ~ path;
1262 		path.normalize();
1263 
1264 		string[string] depVers;
1265 		string[] notFound; // keep track of any failed packages in here
1266 		foreach (dep; deps) {
1267 			Version ver;
1268 			try {
1269 				ver = getLatestVersion(dep);
1270 				depVers[dep] = ver.isBranch ? ver.toString() : "~>" ~ ver.toString();
1271 			} catch (Exception e) {
1272 				notFound ~= dep;
1273 			}
1274 		}
1275 
1276 		if(notFound.length > 1){
1277 			throw new Exception(.format("Couldn't find packages: %-(%s, %).", notFound));
1278 		}
1279 		else if(notFound.length == 1){
1280 			throw new Exception(.format("Couldn't find package: %-(%s, %).", notFound));
1281 		}
1282 
1283 		if (m_dryRun) return;
1284 
1285 		initPackage(path, depVers, type, format, recipe_callback);
1286 
1287 		if (!["vibe.d", "deimos", "minimal"].canFind(type)) {
1288 			runCustomInitialization(path, type, app_args);
1289 		}
1290 
1291 		//Act smug to the user.
1292 		logInfo("Successfully created an empty project in '%s'.", path.toNativeString());
1293 	}
1294 
1295 	private void runCustomInitialization(NativePath path, string type, string[] runArgs)
1296 	{
1297 		string packageName = type;
1298 		auto template_pack = m_packageManager.getBestPackage(packageName, ">=0.0.0");
1299 		if (!template_pack) template_pack = m_packageManager.getBestPackage(packageName, "~master");
1300 		if (!template_pack) {
1301 			logInfo("%s is not present, getting and storing it user wide", packageName);
1302 			template_pack = fetch(packageName, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1303 		}
1304 
1305 		Package initSubPackage = m_packageManager.getSubPackage(template_pack, "init-exec", false);
1306 		auto template_dub = new Dub(null, m_packageSuppliers);
1307 		template_dub.loadPackage(initSubPackage);
1308 		auto compiler_binary = this.defaultCompiler;
1309 
1310 		GeneratorSettings settings;
1311 		settings.config = "application";
1312 		settings.compiler = getCompiler(compiler_binary);
1313 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
1314 		settings.buildType = "debug";
1315 		settings.run = true;
1316 		settings.runArgs = runArgs;
1317 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
1318 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
1319 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
1320 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
1321 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
1322 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
1323 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
1324 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
1325 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
1326 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
1327 		initSubPackage.recipe.buildSettings.workingDirectory = path.toNativeString();
1328 		template_dub.generateProject("build", settings);
1329 	}
1330 
1331 	/** Converts the package recipe of the loaded root package to the given format.
1332 
1333 		Params:
1334 			destination_file_ext = The file extension matching the desired
1335 				format. Possible values are "json" or "sdl".
1336 			print_only = Print the converted recipe instead of writing to disk
1337 	*/
1338 	void convertRecipe(string destination_file_ext, bool print_only = false)
1339 	{
1340 		import std.path : extension;
1341 		import std.stdio : stdout;
1342 		import dub.recipe.io : serializePackageRecipe, writePackageRecipe;
1343 
1344 		if (print_only) {
1345 			auto dst = stdout.lockingTextWriter;
1346 			serializePackageRecipe(dst, m_project.rootPackage.rawRecipe, "dub."~destination_file_ext);
1347 			return;
1348 		}
1349 
1350 		auto srcfile = m_project.rootPackage.recipePath;
1351 		auto srcext = srcfile.head.name.extension;
1352 		if (srcext == "."~destination_file_ext) {
1353 			logInfo("Package format is already %s.", destination_file_ext);
1354 			return;
1355 		}
1356 
1357 		writePackageRecipe(srcfile.parentPath ~ ("dub."~destination_file_ext), m_project.rootPackage.rawRecipe);
1358 		removeFile(srcfile);
1359 	}
1360 
1361 	/** Runs DDOX to generate or serve documentation.
1362 
1363 		Params:
1364 			run = If set to true, serves documentation on a local web server.
1365 				Otherwise generates actual HTML files.
1366 			generate_args = Additional command line arguments to pass to
1367 				"ddox generate-html" or "ddox serve-html".
1368 	*/
1369 	void runDdox(bool run, string[] generate_args = null)
1370 	{
1371 		import std.process : browse;
1372 
1373 		if (m_dryRun) return;
1374 
1375 		// allow to choose a custom ddox tool
1376 		auto tool = m_project.rootPackage.recipe.ddoxTool;
1377 		if (tool.empty) tool = "ddox";
1378 
1379 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
1380 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
1381 		if (!tool_pack) {
1382 			logInfo("%s is not present, getting and storing it user wide", tool);
1383 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1384 		}
1385 
1386 		auto ddox_dub = new Dub(null, m_packageSuppliers);
1387 		ddox_dub.loadPackage(tool_pack.path);
1388 		ddox_dub.upgrade(UpgradeOptions.select);
1389 
1390 		auto compiler_binary = this.defaultCompiler;
1391 
1392 		GeneratorSettings settings;
1393 		settings.config = "application";
1394 		settings.compiler = getCompiler(compiler_binary); // TODO: not using --compiler ???
1395 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
1396 		settings.buildType = "debug";
1397 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
1398 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
1399 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
1400 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
1401 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
1402 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
1403 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
1404 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
1405 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
1406 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
1407 		settings.run = true;
1408 
1409 		auto filterargs = m_project.rootPackage.recipe.ddoxFilterArgs.dup;
1410 		if (filterargs.empty) filterargs = ["--min-protection=Protected", "--only-documented"];
1411 
1412 		settings.runArgs = "filter" ~ filterargs ~ "docs.json";
1413 		ddox_dub.generateProject("build", settings);
1414 
1415 		auto p = tool_pack.path;
1416 		p.endsWithSlash = true;
1417 		auto tool_path = p.toNativeString();
1418 
1419 		if (run) {
1420 			settings.runArgs = ["serve-html", "--navigation-type=ModuleTree", "docs.json", "--web-file-dir="~tool_path~"public"] ~ generate_args;
1421 			browse("http://127.0.0.1:8080/");
1422 		} else {
1423 			settings.runArgs = ["generate-html", "--navigation-type=ModuleTree", "docs.json", "docs"] ~ generate_args;
1424 		}
1425 		ddox_dub.generateProject("build", settings);
1426 
1427 		if (!run) {
1428 			// TODO: ddox should copy those files itself
1429 			version(Windows) runCommand(`xcopy /S /D "`~tool_path~`public\*" docs\`);
1430 			else runCommand("rsync -ru '"~tool_path~"public/' docs/");
1431 		}
1432 	}
1433 
1434 	private void updatePackageSearchPath()
1435 	{
1436 		// TODO: Remove once `overrideSearchPath` is removed
1437 		if (!m_overrideSearchPath.empty) {
1438 			m_packageManager._disableDefaultSearchPaths = true;
1439 			m_packageManager.searchPath = [m_overrideSearchPath];
1440 			return;
1441 		}
1442 
1443 		auto p = environment.get("DUBPATH");
1444 		NativePath[] paths;
1445 
1446 		version(Windows) enum pathsep = ";";
1447 		else enum pathsep = ":";
1448 		if (p.length) paths ~= p.split(pathsep).map!(p => NativePath(p))().array();
1449 		m_packageManager._disableDefaultSearchPaths = false;
1450 		m_packageManager.searchPath = paths;
1451 	}
1452 
1453 	private void determineDefaultCompiler()
1454 	{
1455 		import std.file : thisExePath;
1456 		import std.path : buildPath, dirName, expandTilde, isAbsolute, isDirSeparator;
1457 		import std.range : front;
1458 
1459 		// Env takes precedence
1460 		if (auto envCompiler = environment.get("DC"))
1461 			m_defaultCompiler = envCompiler;
1462 		else
1463 			m_defaultCompiler = m_config.defaultCompiler.expandTilde;
1464 		if (m_defaultCompiler.length && m_defaultCompiler.isAbsolute)
1465 			return;
1466 
1467 		static immutable BinaryPrefix = `$DUB_BINARY_PATH`;
1468 		if(m_defaultCompiler.startsWith(BinaryPrefix))
1469 		{
1470 			m_defaultCompiler = thisExePath().dirName() ~ m_defaultCompiler[BinaryPrefix.length .. $];
1471 			return;
1472 		}
1473 
1474 		if (!find!isDirSeparator(m_defaultCompiler).empty)
1475 			throw new Exception("defaultCompiler specified in a DUB config file cannot use an unqualified relative path:\n\n" ~ m_defaultCompiler ~
1476 			"\n\nUse \"$DUB_BINARY_PATH/../path/you/want\" instead.");
1477 
1478 		version (Windows) enum sep = ";", exe = ".exe";
1479 		version (Posix) enum sep = ":", exe = "";
1480 
1481 		auto compilers = ["dmd", "gdc", "gdmd", "ldc2", "ldmd2"];
1482 		// If a compiler name is specified, look for it next to dub.
1483 		// Otherwise, look for any of the common compilers adjacent to dub.
1484 		if (m_defaultCompiler.length)
1485 		{
1486 			string compilerPath = buildPath(thisExePath().dirName(), m_defaultCompiler ~ exe);
1487 			if (existsFile(compilerPath))
1488 			{
1489 				m_defaultCompiler = compilerPath;
1490 				return;
1491 			}
1492 		}
1493 		else
1494 		{
1495 			auto nextFound = compilers.find!(bin => existsFile(buildPath(thisExePath().dirName(), bin ~ exe)));
1496 			if (!nextFound.empty)
1497 			{
1498 				m_defaultCompiler = buildPath(thisExePath().dirName(),  nextFound.front ~ exe);
1499 				return;
1500 			}
1501 		}
1502 
1503 		// If nothing found next to dub, search the user's PATH, starting
1504 		// with the compiler name from their DUB config file, if specified.
1505 		auto paths = environment.get("PATH", "").splitter(sep).map!NativePath;
1506 		if (m_defaultCompiler.length && paths.canFind!(p => existsFile(p ~ (m_defaultCompiler~exe))))
1507 			return;
1508 		foreach (p; paths) {
1509 			auto res = compilers.find!(bin => existsFile(p ~ (bin~exe)));
1510 			if (!res.empty) {
1511 				m_defaultCompiler = res.front;
1512 				return;
1513 			}
1514 		}
1515 		m_defaultCompiler = compilers[0];
1516 	}
1517 	
1518 	unittest
1519 	{
1520 		import std.path: buildPath, absolutePath;
1521 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
1522 		immutable olddc = environment.get("DC", null);
1523 		immutable oldpath = environment.get("PATH", null);
1524 		immutable testdir = "test-determineDefaultCompiler";
1525 		void repairenv(string name, string var)
1526 		{
1527 			if (var !is null)
1528 				environment[name] = var;
1529 			else if (name in environment)
1530 				environment.remove(name);
1531 		}
1532 		scope (exit) repairenv("DC", olddc);
1533 		scope (exit) repairenv("PATH", oldpath);
1534 		scope (exit) rmdirRecurse(testdir);
1535 		
1536 		version (Windows) enum sep = ";", exe = ".exe";
1537 		version (Posix) enum sep = ":", exe = "";
1538 		
1539 		immutable dmdpath = testdir.buildPath("dmd", "bin");
1540 		immutable ldcpath = testdir.buildPath("ldc", "bin");
1541 		mkdirRecurse(dmdpath);
1542 		mkdirRecurse(ldcpath);
1543 		immutable dmdbin = dmdpath.buildPath("dmd"~exe);
1544 		immutable ldcbin = ldcpath.buildPath("ldc2"~exe);
1545 		std.file.write(dmdbin, null);
1546 		std.file.write(ldcbin, null);
1547 		
1548 		environment["DC"] = dmdbin.absolutePath();
1549 		dub.determineDefaultCompiler();
1550 		assert(dub.m_defaultCompiler == dmdbin.absolutePath());
1551 		
1552 		environment["DC"] = "dmd";
1553 		environment["PATH"] = dmdpath ~ sep ~ ldcpath;
1554 		dub.determineDefaultCompiler();
1555 		assert(dub.m_defaultCompiler == "dmd");
1556 		
1557 		environment["DC"] = "ldc2";
1558 		environment["PATH"] = dmdpath ~ sep ~ ldcpath;
1559 		dub.determineDefaultCompiler();
1560 		assert(dub.m_defaultCompiler == "ldc2");
1561 		
1562 		environment.remove("DC");
1563 		environment["PATH"] = ldcpath ~ sep ~ dmdpath;
1564 		dub.determineDefaultCompiler();
1565 		assert(dub.m_defaultCompiler == "ldc2");
1566 	}
1567 
1568 	private NativePath makeAbsolute(NativePath p) const { return p.absolute ? p : m_rootPath ~ p; }
1569 	private NativePath makeAbsolute(string p) const { return makeAbsolute(NativePath(p)); }
1570 }
1571 
1572 
1573 /// Option flags for `Dub.fetch`
1574 enum FetchOptions
1575 {
1576 	none = 0,
1577 	forceBranchUpgrade = 1<<0,
1578 	usePrerelease = 1<<1,
1579 	forceRemove = 1<<2, /// Deprecated, does nothing.
1580 	printOnly = 1<<3,
1581 }
1582 
1583 /// Option flags for `Dub.upgrade`
1584 enum UpgradeOptions
1585 {
1586 	none = 0,
1587 	upgrade = 1<<1, /// Upgrade existing packages
1588 	preRelease = 1<<2, /// inclde pre-release versions in upgrade
1589 	forceRemove = 1<<3, /// Deprecated, does nothing.
1590 	select = 1<<4, /// Update the dub.selections.json file with the upgraded versions
1591 	dryRun = 1<<5, /// Instead of downloading new packages, just print a message to notify the user of their existence
1592 	/*deprecated*/ printUpgradesOnly = dryRun, /// deprecated, use dryRun instead
1593 	/*deprecated*/ useCachedResult = 1<<6, /// deprecated, has no effect
1594 	noSaveSelections = 1<<7, /// Don't store updated selections on disk
1595 }
1596 
1597 /// Determines which of the default package suppliers are queried for packages.
1598 enum SkipPackageSuppliers {
1599 	none,       /// Uses all configured package suppliers.
1600 	standard,   /// Does not use the default package suppliers (`defaultPackageSuppliers`).
1601 	configured, /// Does not use default suppliers or suppliers configured in DUB's configuration file
1602 	all         /// Uses only manually specified package suppliers.
1603 }
1604 
1605 private class DependencyVersionResolver : DependencyResolver!(Dependency, Dependency) {
1606 	protected {
1607 		Dub m_dub;
1608 		UpgradeOptions m_options;
1609 		Dependency[][string] m_packageVersions;
1610 		Package[string] m_remotePackages;
1611 		SelectedVersions m_selectedVersions;
1612 		Package m_rootPackage;
1613 		bool[string] m_packagesToUpgrade;
1614 		Package[PackageDependency] m_packages;
1615 		TreeNodes[][TreeNode] m_children;
1616 	}
1617 
1618 
1619 	this(Dub dub, UpgradeOptions options)
1620 	{
1621 		m_dub = dub;
1622 		m_options = options;
1623 	}
1624 
1625 	void addPackageToUpgrade(string name)
1626 	{
1627 		m_packagesToUpgrade[name] = true;
1628 	}
1629 
1630 	Dependency[string] resolve(Package root, SelectedVersions selected_versions)
1631 	{
1632 		m_rootPackage = root;
1633 		m_selectedVersions = selected_versions;
1634 		return super.resolve(TreeNode(root.name, Dependency(root.version_)), (m_options & UpgradeOptions.printUpgradesOnly) == 0);
1635 	}
1636 
1637 	protected bool isFixedPackage(string pack)
1638 	{
1639 		return m_packagesToUpgrade !is null && pack !in m_packagesToUpgrade;
1640 	}
1641 
1642 	protected override Dependency[] getAllConfigs(string pack)
1643 	{
1644 		if (auto pvers = pack in m_packageVersions)
1645 			return *pvers;
1646 
1647 		if ((!(m_options & UpgradeOptions.upgrade) || isFixedPackage(pack)) && m_selectedVersions.hasSelectedVersion(pack)) {
1648 			auto ret = [m_selectedVersions.getSelectedVersion(pack)];
1649 			logDiagnostic("Using fixed selection %s %s", pack, ret[0]);
1650 			m_packageVersions[pack] = ret;
1651 			return ret;
1652 		}
1653 
1654 		logDiagnostic("Search for versions of %s (%s package suppliers)", pack, m_dub.m_packageSuppliers.length);
1655 		Version[] versions;
1656 		foreach (p; m_dub.packageManager.getPackageIterator(pack))
1657 			versions ~= p.version_;
1658 
1659 		foreach (ps; m_dub.m_packageSuppliers) {
1660 			try {
1661 				auto vers = ps.getVersions(pack);
1662 				vers.reverse();
1663 				if (!vers.length) {
1664 					logDiagnostic("No versions for %s for %s", pack, ps.description);
1665 					continue;
1666 				}
1667 
1668 				versions ~= vers;
1669 				break;
1670 			} catch (Exception e) {
1671 				logWarn("Package %s not found in %s: %s", pack, ps.description, e.msg);
1672 				logDebug("Full error: %s", e.toString().sanitize);
1673 			}
1674 		}
1675 
1676 		// sort by version, descending, and remove duplicates
1677 		versions = versions.sort!"a>b".uniq.array;
1678 
1679 		// move pre-release versions to the back of the list if no preRelease flag is given
1680 		if (!(m_options & UpgradeOptions.preRelease))
1681 			versions = versions.filter!(v => !v.isPreRelease).array ~ versions.filter!(v => v.isPreRelease).array;
1682 
1683 		// filter out invalid/unreachable dependency specs
1684 		versions = versions.filter!((v) {
1685 				bool valid = getPackage(pack, Dependency(v)) !is null;
1686 				if (!valid) logDiagnostic("Excluding invalid dependency specification %s %s from dependency resolution process.", pack, v);
1687 				return valid;
1688 			}).array;
1689 
1690 		if (!versions.length) logDiagnostic("Nothing found for %s", pack);
1691 		else logDiagnostic("Return for %s: %s", pack, versions);
1692 
1693 		auto ret = versions.map!(v => Dependency(v)).array;
1694 		m_packageVersions[pack] = ret;
1695 		return ret;
1696 	}
1697 
1698 	protected override Dependency[] getSpecificConfigs(string pack, TreeNodes nodes)
1699 	{
1700 		if (!nodes.configs.path.empty || !nodes.configs.repository.empty) {
1701 			if (getPackage(pack, nodes.configs)) return [nodes.configs];
1702 			else return null;
1703 		}
1704 		else return null;
1705 	}
1706 
1707 
1708 	protected override TreeNodes[] getChildren(TreeNode node)
1709 	{
1710 		if (auto pc = node in m_children)
1711 			return *pc;
1712 		auto ret = getChildrenRaw(node);
1713 		m_children[node] = ret;
1714 		return ret;
1715 	}
1716 
1717 	private final TreeNodes[] getChildrenRaw(TreeNode node)
1718 	{
1719 		import std.array : appender;
1720 		auto ret = appender!(TreeNodes[]);
1721 		auto pack = getPackage(node.pack, node.config);
1722 		if (!pack) {
1723 			// this can hapen when the package description contains syntax errors
1724 			logDebug("Invalid package in dependency tree: %s %s", node.pack, node.config);
1725 			return null;
1726 		}
1727 		auto basepack = pack.basePackage;
1728 
1729 		foreach (d; pack.getAllDependenciesRange()) {
1730 			auto dbasename = getBasePackageName(d.name);
1731 
1732 			// detect dependencies to the root package (or sub packages thereof)
1733 			if (dbasename == basepack.name) {
1734 				auto absdeppath = d.spec.mapToPath(pack.path).path;
1735 				absdeppath.endsWithSlash = true;
1736 				auto subpack = m_dub.m_packageManager.getSubPackage(basepack, getSubPackageName(d.name), true);
1737 				if (subpack) {
1738 					auto desireddeppath = basepack.path;
1739 					desireddeppath.endsWithSlash = true;
1740 
1741 					auto altdeppath = d.name == dbasename ? basepack.path : subpack.path;
1742 					altdeppath.endsWithSlash = true;
1743 
1744 					if (!d.spec.path.empty && absdeppath != desireddeppath)
1745 						logWarn("Warning: Sub package %s, referenced by %s %s must be referenced using the path to its base package",
1746 							subpack.name, pack.name, pack.version_);
1747 
1748 					enforce(d.spec.path.empty || absdeppath == desireddeppath || absdeppath == altdeppath,
1749 						format("Dependency from %s to %s uses wrong path: %s vs. %s",
1750 							node.pack, subpack.name, absdeppath.toNativeString(), desireddeppath.toNativeString()));
1751 				}
1752 				ret ~= TreeNodes(d.name, node.config);
1753 				continue;
1754 			}
1755 
1756 			DependencyType dt;
1757 			if (d.spec.optional) {
1758 				if (d.spec.default_) dt = DependencyType.optionalDefault;
1759 				else dt = DependencyType.optional;
1760 			} else dt = DependencyType.required;
1761 
1762 			Dependency dspec = d.spec.mapToPath(pack.path);
1763 
1764 			// if not upgrading, use the selected version
1765 			if (!(m_options & UpgradeOptions.upgrade) && m_selectedVersions && m_selectedVersions.hasSelectedVersion(dbasename))
1766 				dspec = m_selectedVersions.getSelectedVersion(dbasename);
1767 
1768 			// keep selected optional dependencies and avoid non-selected optional-default dependencies by default
1769 			if (m_selectedVersions && !m_selectedVersions.bare) {
1770 				if (dt == DependencyType.optionalDefault && !m_selectedVersions.hasSelectedVersion(dbasename))
1771 					dt = DependencyType.optional;
1772 				else if (dt == DependencyType.optional && m_selectedVersions.hasSelectedVersion(dbasename))
1773 					dt = DependencyType.optionalDefault;
1774 			}
1775 
1776 			ret ~= TreeNodes(d.name, dspec, dt);
1777 		}
1778 		return ret.data;
1779 	}
1780 
1781 	protected override bool matches(Dependency configs, Dependency config)
1782 	{
1783 		if (!configs.path.empty) return configs.path == config.path;
1784 		return configs.merge(config).valid;
1785 	}
1786 
1787 	private Package getPackage(string name, Dependency dep)
1788 	{
1789 		auto key = PackageDependency(name, dep);
1790 		if (auto pp = key in m_packages)
1791 			return *pp;
1792 		auto p = getPackageRaw(name, dep);
1793 		m_packages[key] = p;
1794 		return p;
1795 	}
1796 
1797 	private Package getPackageRaw(string name, Dependency dep)
1798 	{
1799 		auto basename = getBasePackageName(name);
1800 
1801 		// for sub packages, first try to get them from the base package
1802 		if (basename != name) {
1803 			auto subname = getSubPackageName(name);
1804 			auto basepack = getPackage(basename, dep);
1805 			if (!basepack) return null;
1806 			if (auto sp = m_dub.m_packageManager.getSubPackage(basepack, subname, true)) {
1807 				return sp;
1808 			} else if (!basepack.subPackages.canFind!(p => p.path.length)) {
1809 				// note: external sub packages are handled further below
1810 				auto spr = basepack.getInternalSubPackage(subname);
1811 				if (!spr.isNull) {
1812 					auto sp = new Package(spr.get, basepack.path, basepack);
1813 					m_remotePackages[sp.name] = sp;
1814 					return sp;
1815 				} else {
1816 					logDiagnostic("Sub package %s doesn't exist in %s %s.", name, basename, dep.version_);
1817 					return null;
1818 				}
1819 			} else if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) {
1820 				return ret;
1821 			} else {
1822 				logDiagnostic("External sub package %s %s not found.", name, dep.version_);
1823 				return null;
1824 			}
1825 		}
1826 
1827 		// shortcut if the referenced package is the root package
1828 		if (basename == m_rootPackage.basePackage.name)
1829 			return m_rootPackage.basePackage;
1830 
1831 		if (!dep.repository.empty) {
1832 			auto ret = m_dub.packageManager.loadSCMPackage(name, dep);
1833 			return ret !is null && dep.matches(ret.version_) ? ret : null;
1834 		} else if (!dep.path.empty) {
1835 			try {
1836 				auto ret = m_dub.packageManager.getOrLoadPackage(dep.path);
1837 				if (dep.matches(ret.version_)) return ret;
1838 			} catch (Exception e) {
1839 				logDiagnostic("Failed to load path based dependency %s: %s", name, e.msg);
1840 				logDebug("Full error: %s", e.toString().sanitize);
1841 				return null;
1842 			}
1843 		}
1844 
1845 		if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep))
1846 			return ret;
1847 
1848 		auto key = name ~ ":" ~ dep.version_.toString();
1849 		if (auto ret = key in m_remotePackages)
1850 			return *ret;
1851 
1852 		auto prerelease = (m_options & UpgradeOptions.preRelease) != 0;
1853 
1854 		auto rootpack = name.split(":")[0];
1855 
1856 		foreach (ps; m_dub.m_packageSuppliers) {
1857 			if (rootpack == name) {
1858 				try {
1859 					auto desc = ps.fetchPackageRecipe(name, dep, prerelease);
1860 					if (desc.type == Json.Type.null_)
1861 						continue;
1862 					auto ret = new Package(desc);
1863 					m_remotePackages[key] = ret;
1864 					return ret;
1865 				} catch (Exception e) {
1866 					logDiagnostic("Metadata for %s %s could not be downloaded from %s: %s", name, dep, ps.description, e.msg);
1867 					logDebug("Full error: %s", e.toString().sanitize);
1868 				}
1869 			} else {
1870 				logDiagnostic("Package %s not found in base package description (%s). Downloading whole package.", name, dep.version_.toString());
1871 				try {
1872 					FetchOptions fetchOpts;
1873 					fetchOpts |= prerelease ? FetchOptions.usePrerelease : FetchOptions.none;
1874 					m_dub.fetch(rootpack, dep, m_dub.defaultPlacementLocation, fetchOpts, "need sub package description");
1875 					auto ret = m_dub.m_packageManager.getBestPackage(name, dep);
1876 					if (!ret) {
1877 						logWarn("Package %s %s doesn't have a sub package %s", rootpack, dep.version_, name);
1878 						return null;
1879 					}
1880 					m_remotePackages[key] = ret;
1881 					return ret;
1882 				} catch (Exception e) {
1883 					logDiagnostic("Package %s could not be downloaded from %s: %s", rootpack, ps.description, e.msg);
1884 					logDebug("Full error: %s", e.toString().sanitize);
1885 				}
1886 			}
1887 		}
1888 
1889 		m_remotePackages[key] = null;
1890 
1891 		logWarn("Package %s %s could not be loaded either locally, or from the configured package registries.", name, dep);
1892 		return null;
1893 	}
1894 }
1895 
1896 private struct SpecialDirs {
1897 	NativePath temp;
1898 	NativePath userSettings;
1899 	NativePath systemSettings;
1900 	NativePath localRepository;
1901 }
1902 
1903 private class DubConfig {
1904 	private {
1905 		DubConfig m_parentConfig;
1906 		Json m_data;
1907 	}
1908 
1909 	this(Json data, DubConfig parent_config)
1910 	{
1911 		m_data = data;
1912 		m_parentConfig = parent_config;
1913 	}
1914 
1915 	@property string[] registryURLs()
1916 	{
1917 		string[] ret;
1918 		if (auto pv = "registryUrls" in m_data)
1919 			ret = (*pv).deserializeJson!(string[]);
1920 		if (m_parentConfig) ret ~= m_parentConfig.registryURLs;
1921 		return ret;
1922 	}
1923 
1924 	@property SkipPackageSuppliers skipRegistry()
1925 	{
1926 		if(auto pv = "skipRegistry" in m_data)
1927 			return to!SkipPackageSuppliers((*pv).get!string);
1928 
1929 		if (m_parentConfig)
1930 			return m_parentConfig.skipRegistry;
1931 
1932 		return SkipPackageSuppliers.none;
1933 	}
1934 
1935 	@property NativePath[] customCachePaths()
1936 	{
1937 		import std.algorithm.iteration : map;
1938 		import std.array : array;
1939 
1940 		NativePath[] ret;
1941 		if (auto pv = "customCachePaths" in m_data)
1942 			ret = (*pv).deserializeJson!(string[])
1943 				.map!(s => NativePath(s))
1944 				.array;
1945 		if (m_parentConfig)
1946 			ret ~= m_parentConfig.customCachePaths;
1947 		return ret;
1948 	}
1949 
1950 	@property string defaultCompiler()
1951 	const {
1952 		if (auto pv = "defaultCompiler" in m_data)
1953 			return pv.get!string;
1954 		if (m_parentConfig) return m_parentConfig.defaultCompiler;
1955 		return null;
1956 	}
1957 
1958 	@property string defaultArchitecture()
1959 	const {
1960 		if(auto pv = "defaultArchitecture" in m_data)
1961 			return (*pv).get!string;
1962 		if (m_parentConfig) return m_parentConfig.defaultArchitecture;
1963 		return null;
1964 	}
1965 
1966 	@property bool defaultLowMemory()
1967 	const {
1968 		if(auto pv = "defaultLowMemory" in m_data)
1969 			return (*pv).get!bool;
1970 		if (m_parentConfig) return m_parentConfig.defaultLowMemory;
1971 		return false;
1972 	}
1973 	
1974 	@property string[string] defaultEnvironments()
1975 	const {
1976 		if (auto pv = "defaultEnvironments" in m_data)
1977 			return deserializeJson!(string[string])(*cast(Json*)pv);
1978 		if (m_parentConfig) return m_parentConfig.defaultEnvironments;
1979 		return null;
1980 	}
1981 	
1982 	@property string[string] defaultBuildEnvironments()
1983 	const {
1984 		if (auto pv = "defaultBuildEnvironments" in m_data)
1985 			return deserializeJson!(string[string])(*cast(Json*)pv);
1986 		if (m_parentConfig) return m_parentConfig.defaultBuildEnvironments;
1987 		return null;
1988 	}
1989 	
1990 	@property string[string] defaultRunEnvironments()
1991 	const {
1992 		if (auto pv = "defaultRunEnvironments" in m_data)
1993 			return deserializeJson!(string[string])(*cast(Json*)pv);
1994 		if (m_parentConfig) return m_parentConfig.defaultRunEnvironments;
1995 		return null;
1996 	}
1997 	
1998 	@property string[string] defaultPreGenerateEnvironments()
1999 	const {
2000 		if (auto pv = "defaultPreGenerateEnvironments" in m_data)
2001 			return deserializeJson!(string[string])(*cast(Json*)pv);
2002 		if (m_parentConfig) return m_parentConfig.defaultPreGenerateEnvironments;
2003 		return null;
2004 	}
2005 	
2006 	@property string[string] defaultPostGenerateEnvironments()
2007 	const {
2008 		if (auto pv = "defaultPostGenerateEnvironments" in m_data)
2009 			return deserializeJson!(string[string])(*cast(Json*)pv);
2010 		if (m_parentConfig) return m_parentConfig.defaultPostGenerateEnvironments;
2011 		return null;
2012 	}
2013 	
2014 	@property string[string] defaultPreBuildEnvironments()
2015 	const {
2016 		if (auto pv = "defaultPreBuildEnvironments" in m_data)
2017 			return deserializeJson!(string[string])(*cast(Json*)pv);
2018 		if (m_parentConfig) return m_parentConfig.defaultPreBuildEnvironments;
2019 		return null;
2020 	}
2021 	
2022 	@property string[string] defaultPostBuildEnvironments()
2023 	const {
2024 		if (auto pv = "defaultPostBuildEnvironments" in m_data)
2025 			return deserializeJson!(string[string])(*cast(Json*)pv);
2026 		if (m_parentConfig) return m_parentConfig.defaultPostBuildEnvironments;
2027 		return null;
2028 	}
2029 	
2030 	@property string[string] defaultPreRunEnvironments()
2031 	const {
2032 		if (auto pv = "defaultPreRunEnvironments" in m_data)
2033 			return deserializeJson!(string[string])(*cast(Json*)pv);
2034 		if (m_parentConfig) return m_parentConfig.defaultPreRunEnvironments;
2035 		return null;
2036 	}
2037 	
2038 	@property string[string] defaultPostRunEnvironments()
2039 	const {
2040 		if (auto pv = "defaultPostRunEnvironments" in m_data)
2041 			return deserializeJson!(string[string])(*cast(Json*)pv);
2042 		if (m_parentConfig) return m_parentConfig.defaultPostRunEnvironments;
2043 		return null;
2044 	}
2045 }