1 /**
2 	A package manager.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff, 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff, Sönke Ludwig
7 */
8 module dub.dub;
9 
10 import dub.compilers.compiler;
11 import dub.dependency;
12 import dub.dependencyresolver;
13 import dub.internal.utils;
14 import dub.internal.vibecompat.core.file;
15 import dub.internal.vibecompat.core.log;
16 import dub.internal.vibecompat.data.json;
17 import dub.internal.vibecompat.inet.url;
18 import dub.package_;
19 import dub.packagemanager;
20 import dub.packagesuppliers;
21 import dub.project;
22 import dub.generators.generator;
23 import dub.init;
24 
25 import std.algorithm;
26 import std.array : array, replace;
27 import std.conv : to;
28 import std.exception : enforce;
29 import std.file;
30 import std.process : environment;
31 import std.range : assumeSorted, empty;
32 import std.string;
33 import std.encoding : sanitize;
34 
35 // Set output path and options for coverage reports
36 version (DigitalMars) version (D_Coverage)
37 {
38 	shared static this()
39 	{
40 		import core.runtime, std.file, std.path, std.stdio;
41 		dmd_coverSetMerge(true);
42 		auto path = buildPath(dirName(thisExePath()), "../cov");
43 		if (!path.exists)
44 			mkdir(path);
45 		dmd_coverDestPath(path);
46 	}
47 }
48 
49 static this()
50 {
51 	import dub.compilers.dmd : DMDCompiler;
52 	import dub.compilers.gdc : GDCCompiler;
53 	import dub.compilers.ldc : LDCCompiler;
54 	registerCompiler(new DMDCompiler);
55 	registerCompiler(new GDCCompiler);
56 	registerCompiler(new LDCCompiler);
57 }
58 
59 deprecated("use defaultRegistryURLs") enum defaultRegistryURL = defaultRegistryURLs[0];
60 
61 /// The URL to the official package registry and it's default fallback registries.
62 static immutable string[] defaultRegistryURLs = [
63 	"https://code.dlang.org/",
64 	"https://codemirror.dlang.org/",
65 	"https://dub.bytecraft.nl/",
66 	"https://code-mirror.dlang.io/",
67 ];
68 
69 /** Returns a default list of package suppliers.
70 
71 	This will contain a single package supplier that points to the official
72 	package registry.
73 
74 	See_Also: `defaultRegistryURLs`
75 */
76 PackageSupplier[] defaultPackageSuppliers()
77 {
78 	logDiagnostic("Using dub registry url '%s'", defaultRegistryURLs[0]);
79 	return [new FallbackPackageSupplier(defaultRegistryURLs.map!getRegistryPackageSupplier.array)];
80 }
81 
82 /** Returns a registry package supplier according to protocol.
83 
84 	Allowed protocols are dub+http(s):// and maven+http(s)://.
85 */
86 PackageSupplier getRegistryPackageSupplier(string url)
87 {
88 	switch (url.startsWith("dub+", "mvn+", "file://"))
89 	{
90 		case 1:
91 			return new RegistryPackageSupplier(URL(url[4..$]));
92 		case 2:
93 			return new MavenRegistryPackageSupplier(URL(url[4..$]));
94 		case 3:
95 			return new FileSystemPackageSupplier(NativePath(url[7..$]));
96 		default:
97 			return new RegistryPackageSupplier(URL(url));
98 	}
99 }
100 
101 unittest
102 {
103 	auto dubRegistryPackageSupplier = getRegistryPackageSupplier("dub+https://code.dlang.org");
104 	assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org"));
105 
106 	dubRegistryPackageSupplier = getRegistryPackageSupplier("https://code.dlang.org");
107 	assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org"));
108 
109 	auto mavenRegistryPackageSupplier = getRegistryPackageSupplier("mvn+http://localhost:8040/maven/libs-release/dubpackages");
110 	assert(mavenRegistryPackageSupplier.description.canFind(" http://localhost:8040/maven/libs-release/dubpackages"));
111 
112 	auto fileSystemPackageSupplier = getRegistryPackageSupplier("file:///etc/dubpackages");
113 	assert(fileSystemPackageSupplier.description.canFind(" " ~ NativePath("/etc/dubpackages").toNativeString));
114 }
115 
116 /** Provides a high-level entry point for DUB's functionality.
117 
118 	This class provides means to load a certain project (a root package with
119 	all of its dependencies) and to perform high-level operations as found in
120 	the command line interface.
121 */
122 class Dub {
123 	private {
124 		bool m_dryRun = false;
125 		PackageManager m_packageManager;
126 		PackageSupplier[] m_packageSuppliers;
127 		NativePath m_rootPath;
128 		SpecialDirs m_dirs;
129 		DubConfig m_config;
130 		NativePath m_projectPath;
131 		Project m_project;
132 		NativePath m_overrideSearchPath;
133 		string m_defaultCompiler;
134 		string m_defaultArchitecture;
135 		bool m_defaultLowMemory;
136 		string[string] m_defaultEnvironments;
137 		string[string] m_defaultBuildEnvironments;
138 		string[string] m_defaultRunEnvironments;
139 		string[string] m_defaultPreGenerateEnvironments;
140 		string[string] m_defaultPostGenerateEnvironments;
141 		string[string] m_defaultPreBuildEnvironments;
142 		string[string] m_defaultPostBuildEnvironments;
143 		string[string] m_defaultPreRunEnvironments;
144 		string[string] m_defaultPostRunEnvironments;
145 
146 	}
147 
148 	/** The default placement location of fetched packages.
149 
150 		This property can be altered, so that packages which are downloaded as part
151 		of the normal upgrade process are stored in a certain location. This is
152 		how the "--local" and "--system" command line switches operate.
153 	*/
154 	PlacementLocation defaultPlacementLocation = PlacementLocation.user;
155 
156 
157 	/** Initializes the instance for use with a specific root package.
158 
159 		Note that a package still has to be loaded using one of the
160 		`loadPackage` overloads.
161 
162 		Params:
163 			root_path = Path to the root package
164 			additional_package_suppliers = A list of package suppliers to try
165 				before the suppliers found in the configurations files and the
166 				`defaultPackageSuppliers`.
167 			skip_registry = Can be used to skip using the configured package
168 				suppliers, as well as the default suppliers.
169 	*/
170 	this(string root_path = ".", PackageSupplier[] additional_package_suppliers = null,
171 			SkipPackageSuppliers skip_registry = SkipPackageSuppliers.none)
172 	{
173 		m_rootPath = NativePath(root_path);
174 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
175 
176 		init(m_rootPath);
177 
178 		m_packageSuppliers = getPackageSuppliers(additional_package_suppliers, skip_registry);
179 		m_packageManager = new PackageManager(m_rootPath, m_dirs.localRepository, m_dirs.systemSettings);
180 
181 		auto ccps = m_config.customCachePaths;
182 		if (ccps.length)
183 			m_packageManager.customCachePaths = ccps;
184 
185 		updatePackageSearchPath();
186 	}
187 
188 	unittest
189 	{
190 		scope (exit) environment.remove("DUB_REGISTRY");
191 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
192 		assert(dub.m_packageSuppliers.length == 0);
193 		environment["DUB_REGISTRY"] = "http://example.com/";
194 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
195 		assert(dub.m_packageSuppliers.length == 1);
196 		environment["DUB_REGISTRY"] = "http://example.com/;http://foo.com/";
197 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
198 		assert(dub.m_packageSuppliers.length == 2);
199 		dub = new Dub(".", [new RegistryPackageSupplier(URL("http://bar.com/"))], SkipPackageSuppliers.configured);
200 		assert(dub.m_packageSuppliers.length == 3);
201 	}
202 
203 	/** Get the list of package suppliers.
204 
205 		Params:
206 			additional_package_suppliers = A list of package suppliers to try
207 				before the suppliers found in the configurations files and the
208 				`defaultPackageSuppliers`.
209 			skip_registry = Can be used to skip using the configured package
210 				suppliers, as well as the default suppliers.
211 	*/
212 	public PackageSupplier[] getPackageSuppliers(PackageSupplier[] additional_package_suppliers, SkipPackageSuppliers skip_registry)
213 	{
214 		PackageSupplier[] ps = additional_package_suppliers;
215 
216 		if (skip_registry < SkipPackageSuppliers.all)
217 		{
218 			ps ~= environment.get("DUB_REGISTRY", null)
219 				.splitter(";")
220 				.map!(url => getRegistryPackageSupplier(url))
221 				.array;
222 		}
223 
224 		if (skip_registry < SkipPackageSuppliers.configured)
225 		{
226 			ps ~= m_config.registryURLs
227 				.map!(url => getRegistryPackageSupplier(url))
228 				.array;
229 		}
230 
231 		if (skip_registry < SkipPackageSuppliers.standard)
232 			ps ~= defaultPackageSuppliers();
233 
234 		return ps;
235 	}
236 
237 	/// ditto
238 	public PackageSupplier[] getPackageSuppliers(PackageSupplier[] additional_package_suppliers)
239 	{
240 		return getPackageSuppliers(additional_package_suppliers, m_config.skipRegistry);
241 	}
242 
243 	unittest
244 	{
245 		scope (exit) environment.remove("DUB_REGISTRY");
246 		auto dub = new Dub(".", null, SkipPackageSuppliers.none);
247 
248 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("none")]), null);
249 		assert(dub.getPackageSuppliers(null).length == 1);
250 
251 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("configured")]), null);
252 		assert(dub.getPackageSuppliers(null).length == 0);
253 
254 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("standard")]), null);
255 		assert(dub.getPackageSuppliers(null).length == 0);
256 
257 		environment["DUB_REGISTRY"] = "http://example.com/";
258 		assert(dub.getPackageSuppliers(null).length == 1);
259 	}
260 
261 	/** Initializes the instance with a single package search path, without
262 		loading a package.
263 
264 		This constructor corresponds to the "--bare" option of the command line
265 		interface. Use
266 	*/
267 	this(NativePath override_path)
268 	{
269 		init(NativePath());
270 		m_overrideSearchPath = override_path;
271 		m_packageManager = new PackageManager(override_path);
272 	}
273 
274 	private void init(NativePath root_path)
275 	{
276 		import std.file : tempDir;
277 		version(Windows) {
278 			m_dirs.systemSettings = NativePath(environment.get("ProgramData")) ~ "dub/";
279 			immutable appDataDir = environment.get("APPDATA");
280 			m_dirs.userSettings = NativePath(appDataDir) ~ "dub/";
281 			m_dirs.localRepository = NativePath(environment.get("LOCALAPPDATA", appDataDir)) ~ "dub";
282 		} else version(Posix){
283 			m_dirs.systemSettings = NativePath("/var/lib/dub/");
284 			m_dirs.userSettings = NativePath(environment.get("HOME")) ~ ".dub/";
285 			if (!m_dirs.userSettings.absolute)
286 				m_dirs.userSettings = NativePath(getcwd()) ~ m_dirs.userSettings;
287 			m_dirs.localRepository = m_dirs.userSettings;
288 		}
289 
290 		m_dirs.temp = NativePath(tempDir);
291 
292 		m_config = new DubConfig(jsonFromFile(m_dirs.systemSettings ~ "settings.json", true), m_config);
293 		m_config = new DubConfig(jsonFromFile(NativePath(thisExePath).parentPath ~ "../etc/dub/settings.json", true), m_config);
294 		m_config = new DubConfig(jsonFromFile(m_dirs.userSettings ~ "settings.json", true), m_config);
295 
296 		if (!root_path.empty)
297 			m_config = new DubConfig(jsonFromFile(root_path ~ "dub.settings.json", true), m_config);
298 
299 		determineDefaultCompiler();
300 
301 		m_defaultArchitecture = m_config.defaultArchitecture;
302 		m_defaultLowMemory = m_config.defaultLowMemory;
303 		m_defaultEnvironments = m_config.defaultEnvironments;
304 		m_defaultBuildEnvironments = m_config.defaultBuildEnvironments;
305 		m_defaultRunEnvironments = m_config.defaultRunEnvironments;
306 		m_defaultPreGenerateEnvironments = m_config.defaultPreGenerateEnvironments;
307 		m_defaultPostGenerateEnvironments = m_config.defaultPostGenerateEnvironments;
308 		m_defaultPreBuildEnvironments = m_config.defaultPreBuildEnvironments;
309 		m_defaultPostBuildEnvironments = m_config.defaultPostBuildEnvironments;
310 		m_defaultPreRunEnvironments = m_config.defaultPreRunEnvironments;
311 		m_defaultPostRunEnvironments = m_config.defaultPostRunEnvironments;
312 	}
313 
314 	@property void dryRun(bool v) { m_dryRun = v; }
315 
316 	/** Returns the root path (usually the current working directory).
317 	*/
318 	@property NativePath rootPath() const { return m_rootPath; }
319 	/// ditto
320 	@property void rootPath(NativePath root_path)
321 	{
322 		m_rootPath = root_path;
323 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
324 	}
325 
326 	/// Returns the name listed in the dub.json of the current
327 	/// application.
328 	@property string projectName() const { return m_project.name; }
329 
330 	@property NativePath projectPath() const { return m_projectPath; }
331 
332 	@property string[] configurations() const { return m_project.configurations; }
333 
334 	@property inout(PackageManager) packageManager() inout { return m_packageManager; }
335 
336 	@property inout(Project) project() inout { return m_project; }
337 
338 	/** Returns the default compiler binary to use for building D code.
339 
340 		If set, the "defaultCompiler" field of the DUB user or system
341 		configuration file will be used. Otherwise the PATH environment variable
342 		will be searched for files named "dmd", "gdc", "gdmd", "ldc2", "ldmd2"
343 		(in that order, taking into account operating system specific file
344 		extensions) and the first match is returned. If no match is found, "dmd"
345 		will be used.
346 	*/
347 	@property string defaultCompiler() const { return m_defaultCompiler; }
348 
349 	/** Returns the default architecture to use for building D code.
350 
351 		If set, the "defaultArchitecture" field of the DUB user or system
352 		configuration file will be used. Otherwise null will be returned.
353 	*/
354 	@property string defaultArchitecture() const { return m_defaultArchitecture; }
355 
356 	/** Returns the default low memory option to use for building D code.
357 
358 		If set, the "defaultLowMemory" field of the DUB user or system
359 		configuration file will be used. Otherwise false will be returned.
360 	*/
361 	@property bool defaultLowMemory() const { return m_defaultLowMemory; }
362 
363 	@property const(string[string]) defaultEnvironments() const { return m_defaultEnvironments; }
364 	@property const(string[string]) defaultBuildEnvironments() const { return m_defaultBuildEnvironments; }
365 	@property const(string[string]) defaultRunEnvironments() const { return m_defaultRunEnvironments; }
366 	@property const(string[string]) defaultPreGenerateEnvironments() const { return m_defaultPreGenerateEnvironments; }
367 	@property const(string[string]) defaultPostGenerateEnvironments() const { return m_defaultPostGenerateEnvironments; }
368 	@property const(string[string]) defaultPreBuildEnvironments() const { return m_defaultPreBuildEnvironments; }
369 	@property const(string[string]) defaultPostBuildEnvironments() const { return m_defaultPostBuildEnvironments; }
370 	@property const(string[string]) defaultPreRunEnvironments() const { return m_defaultPreRunEnvironments; }
371 	@property const(string[string]) defaultPostRunEnvironments() const { return m_defaultPostRunEnvironments; }
372 
373 	/** Loads the package that resides within the configured `rootPath`.
374 	*/
375 	void loadPackage()
376 	{
377 		loadPackage(m_rootPath);
378 	}
379 
380 	/// Loads the package from the specified path as the main project package.
381 	void loadPackage(NativePath path)
382 	{
383 		m_projectPath = path;
384 		updatePackageSearchPath();
385 		m_project = new Project(m_packageManager, m_projectPath);
386 	}
387 
388 	/// Loads a specific package as the main project package (can be a sub package)
389 	void loadPackage(Package pack)
390 	{
391 		m_projectPath = pack.path;
392 		updatePackageSearchPath();
393 		m_project = new Project(m_packageManager, pack);
394 	}
395 
396 	/** Loads a single file package.
397 
398 		Single-file packages are D files that contain a package receipe comment
399 		at their top. A recipe comment must be a nested `/+ ... +/` style
400 		comment, containing the virtual recipe file name and a colon, followed by the
401 		recipe contents (what would normally be in dub.sdl/dub.json).
402 
403 		Example:
404 		---
405 		/+ dub.sdl:
406 		   name "test"
407 		   dependency "vibe-d" version="~>0.7.29"
408 		+/
409 		import vibe.http.server;
410 
411 		void main()
412 		{
413 			auto settings = new HTTPServerSettings;
414 			settings.port = 8080;
415 			listenHTTP(settings, &hello);
416 		}
417 
418 		void hello(HTTPServerRequest req, HTTPServerResponse res)
419 		{
420 			res.writeBody("Hello, World!");
421 		}
422 		---
423 
424 		The script above can be invoked with "dub --single test.d".
425 	*/
426 	void loadSingleFilePackage(NativePath path)
427 	{
428 		import dub.recipe.io : parsePackageRecipe;
429 		import std.file : mkdirRecurse, readText;
430 		import std.path : baseName, stripExtension;
431 
432 		path = makeAbsolute(path);
433 
434 		string file_content = readText(path.toNativeString());
435 
436 		if (file_content.startsWith("#!")) {
437 			auto idx = file_content.indexOf('\n');
438 			enforce(idx > 0, "The source fine doesn't contain anything but a shebang line.");
439 			file_content = file_content[idx+1 .. $];
440 		}
441 
442 		file_content = file_content.strip();
443 
444 		string recipe_content;
445 
446 		if (file_content.startsWith("/+")) {
447 			file_content = file_content[2 .. $];
448 			auto idx = file_content.indexOf("+/");
449 			enforce(idx >= 0, "Missing \"+/\" to close comment.");
450 			recipe_content = file_content[0 .. idx].strip();
451 		} else throw new Exception("The source file must start with a recipe comment.");
452 
453 		auto nidx = recipe_content.indexOf('\n');
454 
455 		auto idx = recipe_content.indexOf(':');
456 		enforce(idx > 0 && (nidx < 0 || nidx > idx),
457 			"The first line of the recipe comment must list the recipe file name followed by a colon (e.g. \"/+ dub.sdl:\").");
458 		auto recipe_filename = recipe_content[0 .. idx];
459 		recipe_content = recipe_content[idx+1 .. $];
460 		auto recipe_default_package_name = path.toString.baseName.stripExtension.strip;
461 
462 		auto recipe = parsePackageRecipe(recipe_content, recipe_filename, null, recipe_default_package_name);
463 		import dub.internal.vibecompat.core.log; logInfo("parsePackageRecipe %s", recipe_filename);
464 		enforce(recipe.buildSettings.sourceFiles.length == 0, "Single-file packages are not allowed to specify source files.");
465 		enforce(recipe.buildSettings.sourcePaths.length == 0, "Single-file packages are not allowed to specify source paths.");
466 		enforce(recipe.buildSettings.importPaths.length == 0, "Single-file packages are not allowed to specify import paths.");
467 		recipe.buildSettings.sourceFiles[""] = [path.toNativeString()];
468 		recipe.buildSettings.sourcePaths[""] = [];
469 		recipe.buildSettings.importPaths[""] = [];
470 		recipe.buildSettings.mainSourceFile = path.toNativeString();
471 		if (recipe.buildSettings.targetType == TargetType.autodetect)
472 			recipe.buildSettings.targetType = TargetType.executable;
473 
474 		auto pack = new Package(recipe, path.parentPath, null, "~master");
475 		loadPackage(pack);
476 	}
477 	/// ditto
478 	void loadSingleFilePackage(string path)
479 	{
480 		loadSingleFilePackage(NativePath(path));
481 	}
482 
483 	deprecated("Instantiate a Dub instance with the single-argument constructor: `new Dub(path)`")
484 	void overrideSearchPath(NativePath path)
485 	{
486 		if (!path.absolute) path = NativePath(getcwd()) ~ path;
487 		m_overrideSearchPath = path;
488 		updatePackageSearchPath();
489 	}
490 
491 	/** Gets the default configuration for a particular build platform.
492 
493 		This forwards to `Project.getDefaultConfiguration` and requires a
494 		project to be loaded.
495 	*/
496 	string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); }
497 
498 	/** Attempts to upgrade the dependency selection of the loaded project.
499 
500 		Params:
501 			options = Flags that control how the upgrade is carried out
502 			packages_to_upgrade = Optional list of packages. If this list
503 				contains one or more packages, only those packages will
504 				be upgraded. Otherwise, all packages will be upgraded at
505 				once.
506 	*/
507 	void upgrade(UpgradeOptions options, string[] packages_to_upgrade = null)
508 	{
509 		// clear non-existent version selections
510 		if (!(options & UpgradeOptions.upgrade)) {
511 			next_pack:
512 			foreach (p; m_project.selections.selectedPackages) {
513 				auto dep = m_project.selections.getSelectedVersion(p);
514 				if (!dep.path.empty) {
515 					auto path = dep.path;
516 					if (!path.absolute) path = this.rootPath ~ path;
517 					try if (m_packageManager.getOrLoadPackage(path)) continue;
518 					catch (Exception e) { logDebug("Failed to load path based selection: %s", e.toString().sanitize); }
519 				} else if (!dep.repository.empty) {
520 					if (m_packageManager.loadSCMPackage(getBasePackageName(p), dep))
521 						continue;
522 				} else {
523 					if (m_packageManager.getPackage(p, dep.version_)) continue;
524 					foreach (ps; m_packageSuppliers) {
525 						try {
526 							auto versions = ps.getVersions(p);
527 							if (versions.canFind!(v => dep.matches(v)))
528 								continue next_pack;
529 						} catch (Exception e) {
530 							logWarn("Error querying versions for %s, %s: %s", p, ps.description, e.msg);
531 							logDebug("Full error: %s", e.toString().sanitize());
532 						}
533 					}
534 				}
535 
536 				logWarn("Selected package %s %s doesn't exist. Using latest matching version instead.", p, dep);
537 				m_project.selections.deselectVersion(p);
538 			}
539 		}
540 
541 		Dependency[string] versions;
542 		auto resolver = new DependencyVersionResolver(this, options);
543 		foreach (p; packages_to_upgrade)
544 			resolver.addPackageToUpgrade(p);
545 		versions = resolver.resolve(m_project.rootPackage, m_project.selections);
546 
547 		if (options & UpgradeOptions.dryRun) {
548 			bool any = false;
549 			string rootbasename = getBasePackageName(m_project.rootPackage.name);
550 
551 			foreach (p, ver; versions) {
552 				if (!ver.path.empty || !ver.repository.empty) continue;
553 
554 				auto basename = getBasePackageName(p);
555 				if (basename == rootbasename) continue;
556 
557 				if (!m_project.selections.hasSelectedVersion(basename)) {
558 					logInfo("Package %s would be selected with version %s.",
559 						basename, ver);
560 					any = true;
561 					continue;
562 				}
563 				auto sver = m_project.selections.getSelectedVersion(basename);
564 				if (!sver.path.empty || !sver.repository.empty) continue;
565 				if (ver.version_ <= sver.version_) continue;
566 				logInfo("Package %s would be upgraded from %s to %s.",
567 					basename, sver, ver);
568 				any = true;
569 			}
570 			if (any) logInfo("Use \"dub upgrade\" to perform those changes.");
571 			return;
572 		}
573 
574 		foreach (p; versions.byKey) {
575 			auto ver = versions[p]; // Workaround for DMD 2.070.0 AA issue (crashes in aaApply2 if iterating by key+value)
576 			assert(!p.canFind(":"), "Resolved packages contain a sub package!?: "~p);
577 			Package pack;
578 			if (!ver.path.empty) {
579 				try pack = m_packageManager.getOrLoadPackage(ver.path);
580 				catch (Exception e) {
581 					logDebug("Failed to load path based selection: %s", e.toString().sanitize);
582 					continue;
583 				}
584 			} else if (!ver.repository.empty) {
585 				pack = m_packageManager.loadSCMPackage(p, ver);
586 			} else {
587 				pack = m_packageManager.getBestPackage(p, ver);
588 				if (pack && m_packageManager.isManagedPackage(pack)
589 					&& ver.version_.isBranch && (options & UpgradeOptions.upgrade) != 0)
590 				{
591 					// TODO: only re-install if there is actually a new commit available
592 					logInfo("Re-installing branch based dependency %s %s", p, ver.toString());
593 					m_packageManager.remove(pack);
594 					pack = null;
595 				}
596 			}
597 
598 			FetchOptions fetchOpts;
599 			fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none;
600 			if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version");
601 			if ((options & UpgradeOptions.select) && p != m_project.rootPackage.name) {
602 				if (!ver.repository.empty) {
603 					m_project.selections.selectVersionWithRepository(p, ver.repository, ver.versionSpec);
604 				} else if (ver.path.empty) {
605 					m_project.selections.selectVersion(p, ver.version_);
606 				} else {
607 					NativePath relpath = ver.path;
608 					if (relpath.absolute) relpath = relpath.relativeTo(m_project.rootPackage.path);
609 					m_project.selections.selectVersion(p, relpath);
610 				}
611 			}
612 		}
613 
614 		string[] missingDependenciesBeforeReinit = m_project.missingDependencies;
615 		m_project.reinit();
616 
617 		if (!m_project.hasAllDependencies) {
618 			auto resolvedDependencies = setDifference(
619 					assumeSorted(missingDependenciesBeforeReinit),
620 					assumeSorted(m_project.missingDependencies)
621 				);
622 			if (!resolvedDependencies.empty)
623 				upgrade(options, m_project.missingDependencies);
624 		}
625 
626 		if ((options & UpgradeOptions.select) && !(options & (UpgradeOptions.noSaveSelections | UpgradeOptions.dryRun)))
627 			m_project.saveSelections();
628 	}
629 
630 	/** Generate project files for a specified generator.
631 
632 		Any existing project files will be overridden.
633 	*/
634 	void generateProject(string ide, GeneratorSettings settings)
635 	{
636 		auto generator = createProjectGenerator(ide, m_project);
637 		if (m_dryRun) return; // TODO: pass m_dryRun to the generator
638 		generator.generate(settings);
639 	}
640 
641 	/** Executes tests on the current project.
642 
643 		Throws an exception, if unittests failed.
644 	*/
645 	void testProject(GeneratorSettings settings, string config, NativePath custom_main_file)
646 	{
647 		if (!custom_main_file.empty && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file;
648 
649 		if (config.length == 0) {
650 			// if a custom main file was given, favor the first library configuration, so that it can be applied
651 			if (!custom_main_file.empty) config = m_project.getDefaultConfiguration(settings.platform, false);
652 			// else look for a "unittest" configuration
653 			if (!config.length && m_project.rootPackage.configurations.canFind("unittest")) config = "unittest";
654 			// if not found, fall back to the first "library" configuration
655 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, false);
656 			// if still nothing found, use the first executable configuration
657 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, true);
658 		}
659 
660 		auto generator = createProjectGenerator("build", m_project);
661 
662 		auto test_config = format("%s-test-%s", m_project.rootPackage.name.replace(".", "-").replace(":", "-"), config);
663 
664 		BuildSettings lbuildsettings = settings.buildSettings;
665 		m_project.addBuildSettings(lbuildsettings, settings, config, null, true);
666 
667 		if (lbuildsettings.targetType == TargetType.none) {
668 			logInfo(`Configuration '%s' has target type "none". Skipping test.`, config);
669 			return;
670 		}
671 
672 		if (lbuildsettings.targetType == TargetType.executable && config == "unittest") {
673 			logInfo("Running custom 'unittest' configuration.", config);
674 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
675 			settings.config = config;
676 		} else if (lbuildsettings.sourceFiles.empty) {
677 			logInfo(`No source files found in configuration '%s'. Falling back to "dub -b unittest".`, config);
678 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
679 			settings.config = m_project.getDefaultConfiguration(settings.platform);
680 		} else {
681 			import std.algorithm : remove;
682 
683 			logInfo(`Generating test runner configuration '%s' for '%s' (%s).`, test_config, config, lbuildsettings.targetType);
684 
685 			BuildSettingsTemplate tcinfo = m_project.rootPackage.recipe.getConfiguration(config).buildSettings;
686 			tcinfo.targetType = TargetType.executable;
687 			tcinfo.targetName = test_config;
688 
689 			auto mainfil = tcinfo.mainSourceFile;
690 			if (!mainfil.length) mainfil = m_project.rootPackage.recipe.buildSettings.mainSourceFile;
691 
692 			string custommodname;
693 			if (!custom_main_file.empty) {
694 				import std.path;
695 				tcinfo.sourceFiles[""] ~= custom_main_file.relativeTo(m_project.rootPackage.path).toNativeString();
696 				tcinfo.importPaths[""] ~= custom_main_file.parentPath.toNativeString();
697 				custommodname = custom_main_file.head.name.baseName(".d");
698 			}
699 
700 			// prepare the list of tested modules
701 
702 			string[] import_modules;
703 			if (settings.single)
704 				lbuildsettings.importPaths ~= NativePath(mainfil).parentPath.toNativeString;
705 			bool firstTimePackage = true;
706 			foreach (file; lbuildsettings.sourceFiles) {
707 				if (file.endsWith(".d")) {
708 					auto fname = NativePath(file).head.name;
709 					NativePath msf = NativePath(mainfil);
710 					if (msf.absolute)
711 						msf = msf.relativeTo(m_project.rootPackage.path);
712 					if (!settings.single && NativePath(file).relativeTo(m_project.rootPackage.path) == msf) {
713 						logWarn("Excluding main source file %s from test.", mainfil);
714 						tcinfo.excludedSourceFiles[""] ~= mainfil;
715 						continue;
716 					}
717 					if (fname == "package.d") {
718 						if (firstTimePackage) {
719 							firstTimePackage = false;
720 							logDiagnostic("Excluding package.d file from test due to https://issues.dlang.org/show_bug.cgi?id=11847");
721 						}
722 						continue;
723 					}
724 					import_modules ~= dub.internal.utils.determineModuleName(lbuildsettings, NativePath(file), m_project.rootPackage.path);
725 				}
726 			}
727 
728 			NativePath mainfile;
729 			if (settings.tempBuild)
730 				mainfile = getTempFile("dub_test_root", ".d");
731 			else {
732 				import dub.generators.build : computeBuildName;
733 				mainfile = m_project.rootPackage.path ~ format(".dub/code/%s_dub_test_root.d", computeBuildName(test_config, settings, import_modules));
734 			}
735 
736 			mkdirRecurse(mainfile.parentPath.toNativeString());
737 
738 			bool regenerateMainFile = settings.force || !existsFile(mainfile);
739 			auto escapedMainFile = mainfile.toNativeString().replace("$", "$$");
740 			// generate main file
741 			tcinfo.sourceFiles[""] ~= escapedMainFile;
742 			tcinfo.mainSourceFile = escapedMainFile;
743 
744 			if (!m_dryRun && regenerateMainFile) {
745 				auto fil = openFile(mainfile, FileMode.createTrunc);
746 				scope(exit) fil.close();
747 				fil.write("module dub_test_root;\n");
748 				fil.write("import std.typetuple;\n");
749 				foreach (mod; import_modules) fil.write(format("static import %s;\n", mod));
750 				fil.write("alias allModules = TypeTuple!(");
751 				foreach (i, mod; import_modules) {
752 					if (i > 0) fil.write(", ");
753 					fil.write(mod);
754 				}
755 				fil.write(");\n");
756 				if (custommodname.length) {
757 					fil.write(format("import %s;\n", custommodname));
758 				} else {
759 					fil.write(q{
760 						import std.stdio;
761 						import core.runtime;
762 
763 						void main() { writeln("All unit tests have been run successfully."); }
764 						shared static this() {
765 							version (Have_tested) {
766 								import tested;
767 								import core.runtime;
768 								import std.exception;
769 								Runtime.moduleUnitTester = () => true;
770 								enforce(runUnitTests!allModules(new ConsoleTestResultWriter), "Unit tests failed.");
771 							}
772 						}
773 					});
774 				}
775 			}
776 			m_project.rootPackage.recipe.configurations ~= ConfigurationInfo(test_config, tcinfo);
777 			m_project = new Project(m_packageManager, m_project.rootPackage);
778 
779 			settings.config = test_config;
780 		}
781 
782 		generator.generate(settings);
783 	}
784 
785 	/** Executes D-Scanner tests on the current project. **/
786 	void lintProject(string[] args)
787 	{
788 		import std.path : buildPath, buildNormalizedPath;
789 
790 		if (m_dryRun) return;
791 
792 		auto tool = "dscanner";
793 
794 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
795 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
796 		if (!tool_pack) {
797 			logInfo("%s is not present, getting and storing it user wide", tool);
798 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
799 		}
800 
801 		auto dscanner_dub = new Dub(null, m_packageSuppliers);
802 		dscanner_dub.loadPackage(tool_pack.path);
803 		dscanner_dub.upgrade(UpgradeOptions.select);
804 
805 		auto compiler_binary = this.defaultCompiler;
806 
807 		GeneratorSettings settings;
808 		settings.config = "application";
809 		settings.compiler = getCompiler(compiler_binary);
810 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
811 		settings.buildType = "debug";
812 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
813 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
814 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
815 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
816 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
817 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
818 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
819 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
820 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
821 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
822 		settings.run = true;
823 
824 		foreach (dependencyPackage; m_project.dependencies)
825 		{
826 			auto cfgs = m_project.getPackageConfigs(settings.platform, null, true);
827 			auto buildSettings = dependencyPackage.getBuildSettings(settings.platform, cfgs[dependencyPackage.name]);
828 			foreach (importPath; buildSettings.importPaths) {
829 				settings.runArgs ~= ["-I", buildNormalizedPath(dependencyPackage.path.toNativeString(), importPath.idup)];
830 			}
831 		}
832 
833 		string configFilePath = buildPath(m_project.rootPackage.path.toNativeString(), "dscanner.ini");
834 		if (!args.canFind("--config") && exists(configFilePath)) {
835 			settings.runArgs ~= ["--config", configFilePath];
836 		}
837 
838 		settings.runArgs ~= args ~ [m_project.rootPackage.path.toNativeString()];
839 		dscanner_dub.generateProject("build", settings);
840 	}
841 
842 	/** Prints the specified build settings necessary for building the root package.
843 	*/
844 	void listProjectData(GeneratorSettings settings, string[] requestedData, ListBuildSettingsFormat list_type)
845 	{
846 		import std.stdio;
847 		import std.ascii : newline;
848 
849 		// Split comma-separated lists
850 		string[] requestedDataSplit =
851 			requestedData
852 			.map!(a => a.splitter(",").map!strip)
853 			.joiner()
854 			.array();
855 
856 		auto data = m_project.listBuildSettings(settings, requestedDataSplit, list_type);
857 
858 		string delimiter;
859 		final switch (list_type) with (ListBuildSettingsFormat) {
860 			case list: delimiter = newline ~ newline; break;
861 			case listNul: delimiter = "\0\0"; break;
862 			case commandLine: delimiter = " "; break;
863 			case commandLineNul: delimiter = "\0\0"; break;
864 		}
865 
866 		write(data.joiner(delimiter));
867 		if (delimiter != "\0\0") writeln();
868 	}
869 
870 	/// Cleans intermediate/cache files of the given package
871 	void cleanPackage(NativePath path)
872 	{
873 		logInfo("Cleaning package at %s...", path.toNativeString());
874 		enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString());
875 
876 		// TODO: clear target files and copy files
877 
878 		if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString());
879 		if (existsFile(path ~ ".dub/metadata_cache.json")) std.file.remove((path ~ ".dub/metadata_cache.json").toNativeString());
880 
881 		auto p = Package.load(path);
882 		if (p.getBuildSettings().targetType == TargetType.none) {
883 			foreach (sp; p.subPackages.filter!(sp => !sp.path.empty)) {
884 				cleanPackage(path ~ sp.path);
885 			}
886 		}
887 	}
888 
889 	/// Fetches the package matching the dependency and places it in the specified location.
890 	Package fetch(string packageId, const Dependency dep, PlacementLocation location, FetchOptions options, string reason = "")
891 	{
892 		auto basePackageName = getBasePackageName(packageId);
893 		Json pinfo;
894 		PackageSupplier supplier;
895 		foreach(ps; m_packageSuppliers){
896 			try {
897 				pinfo = ps.fetchPackageRecipe(basePackageName, dep, (options & FetchOptions.usePrerelease) != 0);
898 				if (pinfo.type == Json.Type.null_)
899 					continue;
900 				supplier = ps;
901 				break;
902 			} catch(Exception e) {
903 				logWarn("Package %s not found for %s: %s", packageId, ps.description, e.msg);
904 				logDebug("Full error: %s", e.toString().sanitize());
905 			}
906 		}
907 		enforce(pinfo.type != Json.Type.undefined, "No package "~packageId~" was found matching the dependency "~dep.toString());
908 		string ver = pinfo["version"].get!string;
909 
910 		NativePath placement;
911 		final switch (location) {
912 			case PlacementLocation.local: placement = m_rootPath ~ ".dub/packages/"; break;
913 			case PlacementLocation.user: placement = m_dirs.localRepository ~ "packages/"; break;
914 			case PlacementLocation.system: placement = m_dirs.systemSettings ~ "packages/"; break;
915 		}
916 
917 		// always upgrade branch based versions - TODO: actually check if there is a new commit available
918 		Package existing;
919 		try existing = m_packageManager.getPackage(packageId, ver, placement);
920 		catch (Exception e) {
921 			logWarn("Failed to load existing package %s: %s", ver, e.msg);
922 			logDiagnostic("Full error: %s", e.toString().sanitize);
923 		}
924 
925 		if (options & FetchOptions.printOnly) {
926 			if (existing && existing.version_ != Version(ver))
927 				logInfo("A new version for %s is available (%s -> %s). Run \"dub upgrade %s\" to switch.",
928 					packageId, existing.version_, ver, packageId);
929 			return null;
930 		}
931 
932 		if (existing) {
933 			if (!ver.startsWith("~") || !(options & FetchOptions.forceBranchUpgrade) || location == PlacementLocation.local) {
934 				// TODO: support git working trees by performing a "git pull" instead of this
935 				logDiagnostic("Package %s %s (%s) is already present with the latest version, skipping upgrade.",
936 					packageId, ver, placement);
937 				return existing;
938 			} else {
939 				logInfo("Removing %s %s to prepare replacement with a new version.", packageId, ver);
940 				if (!m_dryRun) m_packageManager.remove(existing);
941 			}
942 		}
943 
944 		if (reason.length) logInfo("Fetching %s %s (%s)...", packageId, ver, reason);
945 		else logInfo("Fetching %s %s...", packageId, ver);
946 		if (m_dryRun) return null;
947 
948 		logDebug("Acquiring package zip file");
949 
950 		auto clean_package_version = ver[ver.startsWith("~") ? 1 : 0 .. $];
951 		clean_package_version = clean_package_version.replace("+", "_"); // + has special meaning for Optlink
952 		if (!placement.existsFile())
953 			mkdirRecurse(placement.toNativeString());
954 		NativePath dstpath = placement ~ (basePackageName ~ "-" ~ clean_package_version);
955 		if (!dstpath.existsFile())
956 			mkdirRecurse(dstpath.toNativeString());
957 
958 		// Support libraries typically used with git submodules like ae.
959 		// Such libraries need to have ".." as import path but this can create
960 		// import path leakage.
961 		dstpath = dstpath ~ basePackageName;
962 
963 		import std.datetime : seconds;
964 		auto lock = lockFile(dstpath.toNativeString() ~ ".lock", 30.seconds); // possibly wait for other dub instance
965 		if (dstpath.existsFile())
966 		{
967 			m_packageManager.refresh(false);
968 			return m_packageManager.getPackage(packageId, ver, dstpath);
969 		}
970 
971 		// repeat download on corrupted zips, see #1336
972 		foreach_reverse (i; 0..3)
973 		{
974 			import std.zip : ZipException;
975 
976 			auto path = getTempFile(basePackageName, ".zip");
977 			supplier.fetchPackage(path, basePackageName, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
978 			scope(exit) std.file.remove(path.toNativeString());
979 			logDiagnostic("Placing to %s...", placement.toNativeString());
980 
981 			try {
982 				m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
983 				return m_packageManager.getPackage(packageId, ver, dstpath);
984 			} catch (ZipException e) {
985 				logInfo("Failed to extract zip archive for %s %s...", packageId, ver);
986 				// rethrow the exception at the end of the loop
987 				if (i == 0)
988 					throw e;
989 			}
990 		}
991 		assert(0, "Should throw a ZipException instead.");
992 	}
993 
994 	/** Removes a specific locally cached package.
995 
996 		This will delete the package files from disk and removes the
997 		corresponding entry from the list of known packages.
998 
999 		Params:
1000 			pack = Package instance to remove
1001 	*/
1002 	void remove(in Package pack)
1003 	{
1004 		logInfo("Removing %s in %s", pack.name, pack.path.toNativeString());
1005 		if (!m_dryRun) m_packageManager.remove(pack);
1006 	}
1007 
1008 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1009 	void remove(in Package pack, bool force_remove)
1010 	{
1011 		remove(pack);
1012 	}
1013 
1014 	/// @see remove(string, string, RemoveLocation)
1015 	enum RemoveVersionWildcard = "*";
1016 
1017 	/** Removes one or more versions of a locally cached package.
1018 
1019 		This will remove a given package with a specified version from the
1020 		given location. It will remove at most one package, unless `version_`
1021 		is set to `RemoveVersionWildcard`.
1022 
1023 		Params:
1024 			package_id = Name of the package to be removed
1025 			location_ = Specifies the location to look for the given package
1026 				name/version.
1027 			resolve_version = Callback to select package version.
1028 	*/
1029 	void remove(string package_id, PlacementLocation location,
1030 				scope size_t delegate(in Package[] packages) resolve_version)
1031 	{
1032 		enforce(!package_id.empty);
1033 		if (location == PlacementLocation.local) {
1034 			logInfo("To remove a locally placed package, make sure you don't have any data"
1035 					~ "\nleft in it's directory and then simply remove the whole directory.");
1036 			throw new Exception("dub cannot remove locally installed packages.");
1037 		}
1038 
1039 		Package[] packages;
1040 
1041 		// Retrieve packages to be removed.
1042 		foreach(pack; m_packageManager.getPackageIterator(package_id))
1043 			if (m_packageManager.isManagedPackage(pack))
1044 				packages ~= pack;
1045 
1046 		// Check validity of packages to be removed.
1047 		if(packages.empty) {
1048 			throw new Exception("Cannot find package to remove. ("
1049 				~ "id: '" ~ package_id ~ "', location: '" ~ to!string(location) ~ "'"
1050 				~ ")");
1051 		}
1052 
1053 		// Sort package list in ascending version order
1054 		packages.sort!((a, b) => a.version_ < b.version_);
1055 
1056 		immutable idx = resolve_version(packages);
1057 		if (idx == size_t.max)
1058 			return;
1059 		else if (idx != packages.length)
1060 			packages = packages[idx .. idx + 1];
1061 
1062 		logDebug("Removing %s packages.", packages.length);
1063 		foreach(pack; packages) {
1064 			try {
1065 				remove(pack);
1066 				logInfo("Removed %s, version %s.", package_id, pack.version_);
1067 			} catch (Exception e) {
1068 				logError("Failed to remove %s %s: %s", package_id, pack.version_, e.msg);
1069 				logInfo("Continuing with other packages (if any).");
1070 			}
1071 		}
1072 	}
1073 
1074 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1075 	void remove(string package_id, PlacementLocation location, bool force_remove,
1076 				scope size_t delegate(in Package[] packages) resolve_version)
1077 	{
1078 		remove(package_id, location, resolve_version);
1079 	}
1080 
1081 	/** Removes a specific version of a package.
1082 
1083 		Params:
1084 			package_id = Name of the package to be removed
1085 			version_ = Identifying a version or a wild card. If an empty string
1086 				is passed, the package will be removed from the location, if
1087 				there is only one version retrieved. This will throw an
1088 				exception, if there are multiple versions retrieved.
1089 			location_ = Specifies the location to look for the given package
1090 				name/version.
1091 	 */
1092 	void remove(string package_id, string version_, PlacementLocation location)
1093 	{
1094 		remove(package_id, location, (in packages) {
1095 			if (version_ == RemoveVersionWildcard || version_.empty)
1096 				return packages.length;
1097 
1098 			foreach (i, p; packages) {
1099 				if (p.version_ == Version(version_))
1100 					return i;
1101 			}
1102 			throw new Exception("Cannot find package to remove. ("
1103 				~ "id: '" ~ package_id ~ "', version: '" ~ version_ ~ "', location: '" ~ to!string(location) ~ "'"
1104 				~ ")");
1105 		});
1106 	}
1107 
1108 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1109 	void remove(string package_id, string version_, PlacementLocation location, bool force_remove)
1110 	{
1111 		remove(package_id, version_, location);
1112 	}
1113 
1114 	/** Adds a directory to the list of locally known packages.
1115 
1116 		Forwards to `PackageManager.addLocalPackage`.
1117 
1118 		Params:
1119 			path = Path to the package
1120 			ver = Optional version to associate with the package (can be left
1121 				empty)
1122 			system = Make the package known system wide instead of user wide
1123 				(requires administrator privileges).
1124 
1125 		See_Also: `removeLocalPackage`
1126 	*/
1127 	void addLocalPackage(string path, string ver, bool system)
1128 	{
1129 		if (m_dryRun) return;
1130 		m_packageManager.addLocalPackage(makeAbsolute(path), ver, system ? LocalPackageType.system : LocalPackageType.user);
1131 	}
1132 
1133 	/** Removes a directory from the list of locally known packages.
1134 
1135 		Forwards to `PackageManager.removeLocalPackage`.
1136 
1137 		Params:
1138 			path = Path to the package
1139 			system = Make the package known system wide instead of user wide
1140 				(requires administrator privileges).
1141 
1142 		See_Also: `addLocalPackage`
1143 	*/
1144 	void removeLocalPackage(string path, bool system)
1145 	{
1146 		if (m_dryRun) return;
1147 		m_packageManager.removeLocalPackage(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1148 	}
1149 
1150 	/** Registers a local directory to search for packages to use for satisfying
1151 		dependencies.
1152 
1153 		Params:
1154 			path = Path to a directory containing package directories
1155 			system = Make the package known system wide instead of user wide
1156 				(requires administrator privileges).
1157 
1158 		See_Also: `removeSearchPath`
1159 	*/
1160 	void addSearchPath(string path, bool system)
1161 	{
1162 		if (m_dryRun) return;
1163 		m_packageManager.addSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1164 	}
1165 
1166 	/** Unregisters a local directory search path.
1167 
1168 		Params:
1169 			path = Path to a directory containing package directories
1170 			system = Make the package known system wide instead of user wide
1171 				(requires administrator privileges).
1172 
1173 		See_Also: `addSearchPath`
1174 	*/
1175 	void removeSearchPath(string path, bool system)
1176 	{
1177 		if (m_dryRun) return;
1178 		m_packageManager.removeSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1179 	}
1180 
1181 	/** Queries all package suppliers with the given query string.
1182 
1183 		Returns a list of tuples, where the first entry is the human readable
1184 		name of the package supplier and the second entry is the list of
1185 		matched packages.
1186 
1187 		See_Also: `PackageSupplier.searchPackages`
1188 	*/
1189 	auto searchPackages(string query)
1190 	{
1191 		import std.typecons : Tuple, tuple;
1192 		Tuple!(string, PackageSupplier.SearchResult[])[] results;
1193 		foreach (ps; this.m_packageSuppliers) {
1194 			try
1195 				results ~= tuple(ps.description, ps.searchPackages(query));
1196 			catch (Exception e) {
1197 				logWarn("Searching %s for '%s' failed: %s", ps.description, query, e.msg);
1198 			}
1199 		}
1200 		return results.filter!(tup => tup[1].length);
1201 	}
1202 
1203 	/** Returns a list of all available versions (including branches) for a
1204 		particular package.
1205 
1206 		The list returned is based on the registered package suppliers. Local
1207 		packages are not queried in the search for versions.
1208 
1209 		See_also: `getLatestVersion`
1210 	*/
1211 	Version[] listPackageVersions(string name)
1212 	{
1213 		Version[] versions;
1214 		auto basePackageName = getBasePackageName(name);
1215 		foreach (ps; this.m_packageSuppliers) {
1216 			try versions ~= ps.getVersions(basePackageName);
1217 			catch (Exception e) {
1218 				logWarn("Failed to get versions for package %s on provider %s: %s", name, ps.description, e.msg);
1219 			}
1220 		}
1221 		return versions.sort().uniq.array;
1222 	}
1223 
1224 	/** Returns the latest available version for a particular package.
1225 
1226 		This function returns the latest numbered version of a package. If no
1227 		numbered versions are available, it will return an available branch,
1228 		preferring "~master".
1229 
1230 		Params:
1231 			package_name: The name of the package in question.
1232 			prefer_stable: If set to `true` (the default), returns the latest
1233 				stable version, even if there are newer pre-release versions.
1234 
1235 		See_also: `listPackageVersions`
1236 	*/
1237 	Version getLatestVersion(string package_name, bool prefer_stable = true)
1238 	{
1239 		auto vers = listPackageVersions(package_name);
1240 		enforce(!vers.empty, "Failed to find any valid versions for a package name of '"~package_name~"'.");
1241 		auto final_versions = vers.filter!(v => !v.isBranch && !v.isPreRelease).array;
1242 		if (prefer_stable && final_versions.length) return final_versions[$-1];
1243 		else return vers[$-1];
1244 	}
1245 
1246 	/** Initializes a directory with a package skeleton.
1247 
1248 		Params:
1249 			path = Path of the directory to create the new package in. The
1250 				directory will be created if it doesn't exist.
1251 			deps = List of dependencies to add to the package recipe.
1252 			type = Specifies the type of the application skeleton to use.
1253 			format = Determines the package recipe format to use.
1254 			recipe_callback = Optional callback that can be used to
1255 				customize the recipe before it gets written.
1256 	*/
1257 	void createEmptyPackage(NativePath path, string[] deps, string type,
1258 		PackageFormat format = PackageFormat.sdl,
1259 		scope void delegate(ref PackageRecipe, ref PackageFormat) recipe_callback = null,
1260 		string[] app_args = [])
1261 	{
1262 		if (!path.absolute) path = m_rootPath ~ path;
1263 		path.normalize();
1264 
1265 		string[string] depVers;
1266 		string[] notFound; // keep track of any failed packages in here
1267 		foreach (dep; deps) {
1268 			Version ver;
1269 			try {
1270 				ver = getLatestVersion(dep);
1271 				depVers[dep] = ver.isBranch ? ver.toString() : "~>" ~ ver.toString();
1272 			} catch (Exception e) {
1273 				notFound ~= dep;
1274 			}
1275 		}
1276 
1277 		if(notFound.length > 1){
1278 			throw new Exception(.format("Couldn't find packages: %-(%s, %).", notFound));
1279 		}
1280 		else if(notFound.length == 1){
1281 			throw new Exception(.format("Couldn't find package: %-(%s, %).", notFound));
1282 		}
1283 
1284 		if (m_dryRun) return;
1285 
1286 		initPackage(path, depVers, type, format, recipe_callback);
1287 
1288 		if (!["vibe.d", "deimos", "minimal"].canFind(type)) {
1289 			runCustomInitialization(path, type, app_args);
1290 		}
1291 
1292 		//Act smug to the user.
1293 		logInfo("Successfully created an empty project in '%s'.", path.toNativeString());
1294 	}
1295 
1296 	private void runCustomInitialization(NativePath path, string type, string[] runArgs)
1297 	{
1298 		string packageName = type;
1299 		auto template_pack = m_packageManager.getBestPackage(packageName, ">=0.0.0");
1300 		if (!template_pack) template_pack = m_packageManager.getBestPackage(packageName, "~master");
1301 		if (!template_pack) {
1302 			logInfo("%s is not present, getting and storing it user wide", packageName);
1303 			template_pack = fetch(packageName, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1304 		}
1305 
1306 		Package initSubPackage = m_packageManager.getSubPackage(template_pack, "init-exec", false);
1307 		auto template_dub = new Dub(null, m_packageSuppliers);
1308 		template_dub.loadPackage(initSubPackage);
1309 		auto compiler_binary = this.defaultCompiler;
1310 
1311 		GeneratorSettings settings;
1312 		settings.config = "application";
1313 		settings.compiler = getCompiler(compiler_binary);
1314 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
1315 		settings.buildType = "debug";
1316 		settings.run = true;
1317 		settings.runArgs = runArgs;
1318 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
1319 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
1320 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
1321 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
1322 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
1323 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
1324 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
1325 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
1326 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
1327 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
1328 		initSubPackage.recipe.buildSettings.workingDirectory = path.toNativeString();
1329 		template_dub.generateProject("build", settings);
1330 	}
1331 
1332 	/** Converts the package recipe of the loaded root package to the given format.
1333 
1334 		Params:
1335 			destination_file_ext = The file extension matching the desired
1336 				format. Possible values are "json" or "sdl".
1337 			print_only = Print the converted recipe instead of writing to disk
1338 	*/
1339 	void convertRecipe(string destination_file_ext, bool print_only = false)
1340 	{
1341 		import std.path : extension;
1342 		import std.stdio : stdout;
1343 		import dub.recipe.io : serializePackageRecipe, writePackageRecipe;
1344 
1345 		if (print_only) {
1346 			auto dst = stdout.lockingTextWriter;
1347 			serializePackageRecipe(dst, m_project.rootPackage.rawRecipe, "dub."~destination_file_ext);
1348 			return;
1349 		}
1350 
1351 		auto srcfile = m_project.rootPackage.recipePath;
1352 		auto srcext = srcfile.head.name.extension;
1353 		if (srcext == "."~destination_file_ext) {
1354 			logInfo("Package format is already %s.", destination_file_ext);
1355 			return;
1356 		}
1357 
1358 		writePackageRecipe(srcfile.parentPath ~ ("dub."~destination_file_ext), m_project.rootPackage.rawRecipe);
1359 		removeFile(srcfile);
1360 	}
1361 
1362 	/** Runs DDOX to generate or serve documentation.
1363 
1364 		Params:
1365 			run = If set to true, serves documentation on a local web server.
1366 				Otherwise generates actual HTML files.
1367 			generate_args = Additional command line arguments to pass to
1368 				"ddox generate-html" or "ddox serve-html".
1369 	*/
1370 	void runDdox(bool run, string[] generate_args = null)
1371 	{
1372 		import std.process : browse;
1373 
1374 		if (m_dryRun) return;
1375 
1376 		// allow to choose a custom ddox tool
1377 		auto tool = m_project.rootPackage.recipe.ddoxTool;
1378 		if (tool.empty) tool = "ddox";
1379 
1380 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
1381 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
1382 		if (!tool_pack) {
1383 			logInfo("%s is not present, getting and storing it user wide", tool);
1384 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1385 		}
1386 
1387 		auto ddox_dub = new Dub(null, m_packageSuppliers);
1388 		ddox_dub.loadPackage(tool_pack.path);
1389 		ddox_dub.upgrade(UpgradeOptions.select);
1390 
1391 		auto compiler_binary = this.defaultCompiler;
1392 
1393 		GeneratorSettings settings;
1394 		settings.config = "application";
1395 		settings.compiler = getCompiler(compiler_binary); // TODO: not using --compiler ???
1396 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
1397 		settings.buildType = "debug";
1398 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
1399 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
1400 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
1401 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
1402 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
1403 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
1404 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
1405 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
1406 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
1407 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
1408 		settings.run = true;
1409 
1410 		auto filterargs = m_project.rootPackage.recipe.ddoxFilterArgs.dup;
1411 		if (filterargs.empty) filterargs = ["--min-protection=Protected", "--only-documented"];
1412 
1413 		settings.runArgs = "filter" ~ filterargs ~ "docs.json";
1414 		ddox_dub.generateProject("build", settings);
1415 
1416 		auto p = tool_pack.path;
1417 		p.endsWithSlash = true;
1418 		auto tool_path = p.toNativeString();
1419 
1420 		if (run) {
1421 			settings.runArgs = ["serve-html", "--navigation-type=ModuleTree", "docs.json", "--web-file-dir="~tool_path~"public"] ~ generate_args;
1422 			browse("http://127.0.0.1:8080/");
1423 		} else {
1424 			settings.runArgs = ["generate-html", "--navigation-type=ModuleTree", "docs.json", "docs"] ~ generate_args;
1425 		}
1426 		ddox_dub.generateProject("build", settings);
1427 
1428 		if (!run) {
1429 			// TODO: ddox should copy those files itself
1430 			version(Windows) runCommand(`xcopy /S /D "`~tool_path~`public\*" docs\`);
1431 			else runCommand("rsync -ru '"~tool_path~"public/' docs/");
1432 		}
1433 	}
1434 
1435 	private void updatePackageSearchPath()
1436 	{
1437 		// TODO: Remove once `overrideSearchPath` is removed
1438 		if (!m_overrideSearchPath.empty) {
1439 			m_packageManager._disableDefaultSearchPaths = true;
1440 			m_packageManager.searchPath = [m_overrideSearchPath];
1441 			return;
1442 		}
1443 
1444 		auto p = environment.get("DUBPATH");
1445 		NativePath[] paths;
1446 
1447 		version(Windows) enum pathsep = ";";
1448 		else enum pathsep = ":";
1449 		if (p.length) paths ~= p.split(pathsep).map!(p => NativePath(p))().array();
1450 		m_packageManager._disableDefaultSearchPaths = false;
1451 		m_packageManager.searchPath = paths;
1452 	}
1453 
1454 	private void determineDefaultCompiler()
1455 	{
1456 		import std.file : thisExePath;
1457 		import std.path : buildPath, dirName, expandTilde, isAbsolute, isDirSeparator;
1458 		import std.range : front;
1459 
1460 		// Env takes precedence
1461 		if (auto envCompiler = environment.get("DC"))
1462 			m_defaultCompiler = envCompiler;
1463 		else
1464 			m_defaultCompiler = m_config.defaultCompiler.expandTilde;
1465 		if (m_defaultCompiler.length && m_defaultCompiler.isAbsolute)
1466 			return;
1467 
1468 		static immutable BinaryPrefix = `$DUB_BINARY_PATH`;
1469 		if(m_defaultCompiler.startsWith(BinaryPrefix))
1470 		{
1471 			m_defaultCompiler = thisExePath().dirName() ~ m_defaultCompiler[BinaryPrefix.length .. $];
1472 			return;
1473 		}
1474 
1475 		if (!find!isDirSeparator(m_defaultCompiler).empty)
1476 			throw new Exception("defaultCompiler specified in a DUB config file cannot use an unqualified relative path:\n\n" ~ m_defaultCompiler ~
1477 			"\n\nUse \"$DUB_BINARY_PATH/../path/you/want\" instead.");
1478 
1479 		version (Windows) enum sep = ";", exe = ".exe";
1480 		version (Posix) enum sep = ":", exe = "";
1481 
1482 		auto compilers = ["dmd", "gdc", "gdmd", "ldc2", "ldmd2"];
1483 		// If a compiler name is specified, look for it next to dub.
1484 		// Otherwise, look for any of the common compilers adjacent to dub.
1485 		if (m_defaultCompiler.length)
1486 		{
1487 			string compilerPath = buildPath(thisExePath().dirName(), m_defaultCompiler ~ exe);
1488 			if (existsFile(compilerPath))
1489 			{
1490 				m_defaultCompiler = compilerPath;
1491 				return;
1492 			}
1493 		}
1494 		else
1495 		{
1496 			auto nextFound = compilers.find!(bin => existsFile(buildPath(thisExePath().dirName(), bin ~ exe)));
1497 			if (!nextFound.empty)
1498 			{
1499 				m_defaultCompiler = buildPath(thisExePath().dirName(),  nextFound.front ~ exe);
1500 				return;
1501 			}
1502 		}
1503 
1504 		// If nothing found next to dub, search the user's PATH, starting
1505 		// with the compiler name from their DUB config file, if specified.
1506 		auto paths = environment.get("PATH", "").splitter(sep).map!NativePath;
1507 		if (m_defaultCompiler.length && paths.canFind!(p => existsFile(p ~ (m_defaultCompiler~exe))))
1508 			return;
1509 		foreach (p; paths) {
1510 			auto res = compilers.find!(bin => existsFile(p ~ (bin~exe)));
1511 			if (!res.empty) {
1512 				m_defaultCompiler = res.front;
1513 				return;
1514 			}
1515 		}
1516 		m_defaultCompiler = compilers[0];
1517 	}
1518 
1519 	unittest
1520 	{
1521 		import std.path: buildPath, absolutePath;
1522 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
1523 		immutable olddc = environment.get("DC", null);
1524 		immutable oldpath = environment.get("PATH", null);
1525 		immutable testdir = "test-determineDefaultCompiler";
1526 		void repairenv(string name, string var)
1527 		{
1528 			if (var !is null)
1529 				environment[name] = var;
1530 			else if (name in environment)
1531 				environment.remove(name);
1532 		}
1533 		scope (exit) repairenv("DC", olddc);
1534 		scope (exit) repairenv("PATH", oldpath);
1535 		scope (exit) rmdirRecurse(testdir);
1536 
1537 		version (Windows) enum sep = ";", exe = ".exe";
1538 		version (Posix) enum sep = ":", exe = "";
1539 
1540 		immutable dmdpath = testdir.buildPath("dmd", "bin");
1541 		immutable ldcpath = testdir.buildPath("ldc", "bin");
1542 		mkdirRecurse(dmdpath);
1543 		mkdirRecurse(ldcpath);
1544 		immutable dmdbin = dmdpath.buildPath("dmd"~exe);
1545 		immutable ldcbin = ldcpath.buildPath("ldc2"~exe);
1546 		std.file.write(dmdbin, null);
1547 		std.file.write(ldcbin, null);
1548 
1549 		environment["DC"] = dmdbin.absolutePath();
1550 		dub.determineDefaultCompiler();
1551 		assert(dub.m_defaultCompiler == dmdbin.absolutePath());
1552 
1553 		environment["DC"] = "dmd";
1554 		environment["PATH"] = dmdpath ~ sep ~ ldcpath;
1555 		dub.determineDefaultCompiler();
1556 		assert(dub.m_defaultCompiler == "dmd");
1557 
1558 		environment["DC"] = "ldc2";
1559 		environment["PATH"] = dmdpath ~ sep ~ ldcpath;
1560 		dub.determineDefaultCompiler();
1561 		assert(dub.m_defaultCompiler == "ldc2");
1562 
1563 		environment.remove("DC");
1564 		environment["PATH"] = ldcpath ~ sep ~ dmdpath;
1565 		dub.determineDefaultCompiler();
1566 		assert(dub.m_defaultCompiler == "ldc2");
1567 	}
1568 
1569 	private NativePath makeAbsolute(NativePath p) const { return p.absolute ? p : m_rootPath ~ p; }
1570 	private NativePath makeAbsolute(string p) const { return makeAbsolute(NativePath(p)); }
1571 }
1572 
1573 
1574 /// Option flags for `Dub.fetch`
1575 enum FetchOptions
1576 {
1577 	none = 0,
1578 	forceBranchUpgrade = 1<<0,
1579 	usePrerelease = 1<<1,
1580 	forceRemove = 1<<2, /// Deprecated, does nothing.
1581 	printOnly = 1<<3,
1582 }
1583 
1584 /// Option flags for `Dub.upgrade`
1585 enum UpgradeOptions
1586 {
1587 	none = 0,
1588 	upgrade = 1<<1, /// Upgrade existing packages
1589 	preRelease = 1<<2, /// inclde pre-release versions in upgrade
1590 	forceRemove = 1<<3, /// Deprecated, does nothing.
1591 	select = 1<<4, /// Update the dub.selections.json file with the upgraded versions
1592 	dryRun = 1<<5, /// Instead of downloading new packages, just print a message to notify the user of their existence
1593 	/*deprecated*/ printUpgradesOnly = dryRun, /// deprecated, use dryRun instead
1594 	/*deprecated*/ useCachedResult = 1<<6, /// deprecated, has no effect
1595 	noSaveSelections = 1<<7, /// Don't store updated selections on disk
1596 }
1597 
1598 /// Determines which of the default package suppliers are queried for packages.
1599 enum SkipPackageSuppliers {
1600 	none,       /// Uses all configured package suppliers.
1601 	standard,   /// Does not use the default package suppliers (`defaultPackageSuppliers`).
1602 	configured, /// Does not use default suppliers or suppliers configured in DUB's configuration file
1603 	all         /// Uses only manually specified package suppliers.
1604 }
1605 
1606 private class DependencyVersionResolver : DependencyResolver!(Dependency, Dependency) {
1607 	protected {
1608 		Dub m_dub;
1609 		UpgradeOptions m_options;
1610 		Dependency[][string] m_packageVersions;
1611 		Package[string] m_remotePackages;
1612 		SelectedVersions m_selectedVersions;
1613 		Package m_rootPackage;
1614 		bool[string] m_packagesToUpgrade;
1615 		Package[PackageDependency] m_packages;
1616 		TreeNodes[][TreeNode] m_children;
1617 	}
1618 
1619 
1620 	this(Dub dub, UpgradeOptions options)
1621 	{
1622 		m_dub = dub;
1623 		m_options = options;
1624 	}
1625 
1626 	void addPackageToUpgrade(string name)
1627 	{
1628 		m_packagesToUpgrade[name] = true;
1629 	}
1630 
1631 	Dependency[string] resolve(Package root, SelectedVersions selected_versions)
1632 	{
1633 		m_rootPackage = root;
1634 		m_selectedVersions = selected_versions;
1635 		return super.resolve(TreeNode(root.name, Dependency(root.version_)), (m_options & UpgradeOptions.printUpgradesOnly) == 0);
1636 	}
1637 
1638 	protected bool isFixedPackage(string pack)
1639 	{
1640 		return m_packagesToUpgrade !is null && pack !in m_packagesToUpgrade;
1641 	}
1642 
1643 	protected override Dependency[] getAllConfigs(string pack)
1644 	{
1645 		if (auto pvers = pack in m_packageVersions)
1646 			return *pvers;
1647 
1648 		if ((!(m_options & UpgradeOptions.upgrade) || isFixedPackage(pack)) && m_selectedVersions.hasSelectedVersion(pack)) {
1649 			auto ret = [m_selectedVersions.getSelectedVersion(pack)];
1650 			logDiagnostic("Using fixed selection %s %s", pack, ret[0]);
1651 			m_packageVersions[pack] = ret;
1652 			return ret;
1653 		}
1654 
1655 		logDiagnostic("Search for versions of %s (%s package suppliers)", pack, m_dub.m_packageSuppliers.length);
1656 		Version[] versions;
1657 		foreach (p; m_dub.packageManager.getPackageIterator(pack))
1658 			versions ~= p.version_;
1659 
1660 		foreach (ps; m_dub.m_packageSuppliers) {
1661 			try {
1662 				auto vers = ps.getVersions(pack);
1663 				vers.reverse();
1664 				if (!vers.length) {
1665 					logDiagnostic("No versions for %s for %s", pack, ps.description);
1666 					continue;
1667 				}
1668 
1669 				versions ~= vers;
1670 				break;
1671 			} catch (Exception e) {
1672 				logWarn("Package %s not found in %s: %s", pack, ps.description, e.msg);
1673 				logDebug("Full error: %s", e.toString().sanitize);
1674 			}
1675 		}
1676 
1677 		// sort by version, descending, and remove duplicates
1678 		versions = versions.sort!"a>b".uniq.array;
1679 
1680 		// move pre-release versions to the back of the list if no preRelease flag is given
1681 		if (!(m_options & UpgradeOptions.preRelease))
1682 			versions = versions.filter!(v => !v.isPreRelease).array ~ versions.filter!(v => v.isPreRelease).array;
1683 
1684 		// filter out invalid/unreachable dependency specs
1685 		versions = versions.filter!((v) {
1686 				bool valid = getPackage(pack, Dependency(v)) !is null;
1687 				if (!valid) logDiagnostic("Excluding invalid dependency specification %s %s from dependency resolution process.", pack, v);
1688 				return valid;
1689 			}).array;
1690 
1691 		if (!versions.length) logDiagnostic("Nothing found for %s", pack);
1692 		else logDiagnostic("Return for %s: %s", pack, versions);
1693 
1694 		auto ret = versions.map!(v => Dependency(v)).array;
1695 		m_packageVersions[pack] = ret;
1696 		return ret;
1697 	}
1698 
1699 	protected override Dependency[] getSpecificConfigs(string pack, TreeNodes nodes)
1700 	{
1701 		if (!nodes.configs.path.empty || !nodes.configs.repository.empty) {
1702 			if (getPackage(pack, nodes.configs)) return [nodes.configs];
1703 			else return null;
1704 		}
1705 		else return null;
1706 	}
1707 
1708 
1709 	protected override TreeNodes[] getChildren(TreeNode node)
1710 	{
1711 		if (auto pc = node in m_children)
1712 			return *pc;
1713 		auto ret = getChildrenRaw(node);
1714 		m_children[node] = ret;
1715 		return ret;
1716 	}
1717 
1718 	private final TreeNodes[] getChildrenRaw(TreeNode node)
1719 	{
1720 		import std.array : appender;
1721 		auto ret = appender!(TreeNodes[]);
1722 		auto pack = getPackage(node.pack, node.config);
1723 		if (!pack) {
1724 			// this can hapen when the package description contains syntax errors
1725 			logDebug("Invalid package in dependency tree: %s %s", node.pack, node.config);
1726 			return null;
1727 		}
1728 		auto basepack = pack.basePackage;
1729 
1730 		foreach (d; pack.getAllDependenciesRange()) {
1731 			auto dbasename = getBasePackageName(d.name);
1732 
1733 			// detect dependencies to the root package (or sub packages thereof)
1734 			if (dbasename == basepack.name) {
1735 				auto absdeppath = d.spec.mapToPath(pack.path).path;
1736 				absdeppath.endsWithSlash = true;
1737 				auto subpack = m_dub.m_packageManager.getSubPackage(basepack, getSubPackageName(d.name), true);
1738 				if (subpack) {
1739 					auto desireddeppath = basepack.path;
1740 					desireddeppath.endsWithSlash = true;
1741 
1742 					auto altdeppath = d.name == dbasename ? basepack.path : subpack.path;
1743 					altdeppath.endsWithSlash = true;
1744 
1745 					if (!d.spec.path.empty && absdeppath != desireddeppath)
1746 						logWarn("Warning: Sub package %s, referenced by %s %s must be referenced using the path to its base package",
1747 							subpack.name, pack.name, pack.version_);
1748 
1749 					enforce(d.spec.path.empty || absdeppath == desireddeppath || absdeppath == altdeppath,
1750 						format("Dependency from %s to %s uses wrong path: %s vs. %s",
1751 							node.pack, subpack.name, absdeppath.toNativeString(), desireddeppath.toNativeString()));
1752 				}
1753 				ret ~= TreeNodes(d.name, node.config);
1754 				continue;
1755 			}
1756 
1757 			DependencyType dt;
1758 			if (d.spec.optional) {
1759 				if (d.spec.default_) dt = DependencyType.optionalDefault;
1760 				else dt = DependencyType.optional;
1761 			} else dt = DependencyType.required;
1762 
1763 			Dependency dspec = d.spec.mapToPath(pack.path);
1764 
1765 			// if not upgrading, use the selected version
1766 			if (!(m_options & UpgradeOptions.upgrade) && m_selectedVersions && m_selectedVersions.hasSelectedVersion(dbasename))
1767 				dspec = m_selectedVersions.getSelectedVersion(dbasename);
1768 
1769 			// keep selected optional dependencies and avoid non-selected optional-default dependencies by default
1770 			if (m_selectedVersions && !m_selectedVersions.bare) {
1771 				if (dt == DependencyType.optionalDefault && !m_selectedVersions.hasSelectedVersion(dbasename))
1772 					dt = DependencyType.optional;
1773 				else if (dt == DependencyType.optional && m_selectedVersions.hasSelectedVersion(dbasename))
1774 					dt = DependencyType.optionalDefault;
1775 			}
1776 
1777 			ret ~= TreeNodes(d.name, dspec, dt);
1778 		}
1779 		return ret.data;
1780 	}
1781 
1782 	protected override bool matches(Dependency configs, Dependency config)
1783 	{
1784 		if (!configs.path.empty) return configs.path == config.path;
1785 		return configs.merge(config).valid;
1786 	}
1787 
1788 	private Package getPackage(string name, Dependency dep)
1789 	{
1790 		auto key = PackageDependency(name, dep);
1791 		if (auto pp = key in m_packages)
1792 			return *pp;
1793 		auto p = getPackageRaw(name, dep);
1794 		m_packages[key] = p;
1795 		return p;
1796 	}
1797 
1798 	private Package getPackageRaw(string name, Dependency dep)
1799 	{
1800 		auto basename = getBasePackageName(name);
1801 
1802 		// for sub packages, first try to get them from the base package
1803 		if (basename != name) {
1804 			auto subname = getSubPackageName(name);
1805 			auto basepack = getPackage(basename, dep);
1806 			if (!basepack) return null;
1807 			if (auto sp = m_dub.m_packageManager.getSubPackage(basepack, subname, true)) {
1808 				return sp;
1809 			} else if (!basepack.subPackages.canFind!(p => p.path.length)) {
1810 				// note: external sub packages are handled further below
1811 				auto spr = basepack.getInternalSubPackage(subname);
1812 				if (!spr.isNull) {
1813 					auto sp = new Package(spr.get, basepack.path, basepack);
1814 					m_remotePackages[sp.name] = sp;
1815 					return sp;
1816 				} else {
1817 					logDiagnostic("Sub package %s doesn't exist in %s %s.", name, basename, dep.version_);
1818 					return null;
1819 				}
1820 			} else if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) {
1821 				return ret;
1822 			} else {
1823 				logDiagnostic("External sub package %s %s not found.", name, dep.version_);
1824 				return null;
1825 			}
1826 		}
1827 
1828 		// shortcut if the referenced package is the root package
1829 		if (basename == m_rootPackage.basePackage.name)
1830 			return m_rootPackage.basePackage;
1831 
1832 		if (!dep.repository.empty) {
1833 			auto ret = m_dub.packageManager.loadSCMPackage(name, dep);
1834 			return ret !is null && dep.matches(ret.version_) ? ret : null;
1835 		} else if (!dep.path.empty) {
1836 			try {
1837 				auto ret = m_dub.packageManager.getOrLoadPackage(dep.path);
1838 				if (dep.matches(ret.version_)) return ret;
1839 			} catch (Exception e) {
1840 				logDiagnostic("Failed to load path based dependency %s: %s", name, e.msg);
1841 				logDebug("Full error: %s", e.toString().sanitize);
1842 				return null;
1843 			}
1844 		}
1845 
1846 		if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep))
1847 			return ret;
1848 
1849 		auto key = name ~ ":" ~ dep.version_.toString();
1850 		if (auto ret = key in m_remotePackages)
1851 			return *ret;
1852 
1853 		auto prerelease = (m_options & UpgradeOptions.preRelease) != 0;
1854 
1855 		auto rootpack = name.split(":")[0];
1856 
1857 		foreach (ps; m_dub.m_packageSuppliers) {
1858 			if (rootpack == name) {
1859 				try {
1860 					auto desc = ps.fetchPackageRecipe(name, dep, prerelease);
1861 					if (desc.type == Json.Type.null_)
1862 						continue;
1863 					auto ret = new Package(desc);
1864 					m_remotePackages[key] = ret;
1865 					return ret;
1866 				} catch (Exception e) {
1867 					logDiagnostic("Metadata for %s %s could not be downloaded from %s: %s", name, dep, ps.description, e.msg);
1868 					logDebug("Full error: %s", e.toString().sanitize);
1869 				}
1870 			} else {
1871 				logDiagnostic("Package %s not found in base package description (%s). Downloading whole package.", name, dep.version_.toString());
1872 				try {
1873 					FetchOptions fetchOpts;
1874 					fetchOpts |= prerelease ? FetchOptions.usePrerelease : FetchOptions.none;
1875 					m_dub.fetch(rootpack, dep, m_dub.defaultPlacementLocation, fetchOpts, "need sub package description");
1876 					auto ret = m_dub.m_packageManager.getBestPackage(name, dep);
1877 					if (!ret) {
1878 						logWarn("Package %s %s doesn't have a sub package %s", rootpack, dep.version_, name);
1879 						return null;
1880 					}
1881 					m_remotePackages[key] = ret;
1882 					return ret;
1883 				} catch (Exception e) {
1884 					logDiagnostic("Package %s could not be downloaded from %s: %s", rootpack, ps.description, e.msg);
1885 					logDebug("Full error: %s", e.toString().sanitize);
1886 				}
1887 			}
1888 		}
1889 
1890 		m_remotePackages[key] = null;
1891 
1892 		logWarn("Package %s %s could not be loaded either locally, or from the configured package registries.", name, dep);
1893 		return null;
1894 	}
1895 }
1896 
1897 private struct SpecialDirs {
1898 	NativePath temp;
1899 	NativePath userSettings;
1900 	NativePath systemSettings;
1901 	NativePath localRepository;
1902 }
1903 
1904 private class DubConfig {
1905 	private {
1906 		DubConfig m_parentConfig;
1907 		Json m_data;
1908 	}
1909 
1910 	this(Json data, DubConfig parent_config)
1911 	{
1912 		m_data = data;
1913 		m_parentConfig = parent_config;
1914 	}
1915 
1916 	@property string[] registryURLs()
1917 	{
1918 		string[] ret;
1919 		if (auto pv = "registryUrls" in m_data)
1920 			ret = (*pv).deserializeJson!(string[]);
1921 		if (m_parentConfig) ret ~= m_parentConfig.registryURLs;
1922 		return ret;
1923 	}
1924 
1925 	@property SkipPackageSuppliers skipRegistry()
1926 	{
1927 		if(auto pv = "skipRegistry" in m_data)
1928 			return to!SkipPackageSuppliers((*pv).get!string);
1929 
1930 		if (m_parentConfig)
1931 			return m_parentConfig.skipRegistry;
1932 
1933 		return SkipPackageSuppliers.none;
1934 	}
1935 
1936 	@property NativePath[] customCachePaths()
1937 	{
1938 		import std.algorithm.iteration : map;
1939 		import std.array : array;
1940 
1941 		NativePath[] ret;
1942 		if (auto pv = "customCachePaths" in m_data)
1943 			ret = (*pv).deserializeJson!(string[])
1944 				.map!(s => NativePath(s))
1945 				.array;
1946 		if (m_parentConfig)
1947 			ret ~= m_parentConfig.customCachePaths;
1948 		return ret;
1949 	}
1950 
1951 	@property string defaultCompiler()
1952 	const {
1953 		if (auto pv = "defaultCompiler" in m_data)
1954 			return pv.get!string;
1955 		if (m_parentConfig) return m_parentConfig.defaultCompiler;
1956 		return null;
1957 	}
1958 
1959 	@property string defaultArchitecture()
1960 	const {
1961 		if(auto pv = "defaultArchitecture" in m_data)
1962 			return (*pv).get!string;
1963 		if (m_parentConfig) return m_parentConfig.defaultArchitecture;
1964 		return null;
1965 	}
1966 
1967 	@property bool defaultLowMemory()
1968 	const {
1969 		if(auto pv = "defaultLowMemory" in m_data)
1970 			return (*pv).get!bool;
1971 		if (m_parentConfig) return m_parentConfig.defaultLowMemory;
1972 		return false;
1973 	}
1974 
1975 	@property string[string] defaultEnvironments()
1976 	const {
1977 		if (auto pv = "defaultEnvironments" in m_data)
1978 			return deserializeJson!(string[string])(*cast(Json*)pv);
1979 		if (m_parentConfig) return m_parentConfig.defaultEnvironments;
1980 		return null;
1981 	}
1982 
1983 	@property string[string] defaultBuildEnvironments()
1984 	const {
1985 		if (auto pv = "defaultBuildEnvironments" in m_data)
1986 			return deserializeJson!(string[string])(*cast(Json*)pv);
1987 		if (m_parentConfig) return m_parentConfig.defaultBuildEnvironments;
1988 		return null;
1989 	}
1990 
1991 	@property string[string] defaultRunEnvironments()
1992 	const {
1993 		if (auto pv = "defaultRunEnvironments" in m_data)
1994 			return deserializeJson!(string[string])(*cast(Json*)pv);
1995 		if (m_parentConfig) return m_parentConfig.defaultRunEnvironments;
1996 		return null;
1997 	}
1998 
1999 	@property string[string] defaultPreGenerateEnvironments()
2000 	const {
2001 		if (auto pv = "defaultPreGenerateEnvironments" in m_data)
2002 			return deserializeJson!(string[string])(*cast(Json*)pv);
2003 		if (m_parentConfig) return m_parentConfig.defaultPreGenerateEnvironments;
2004 		return null;
2005 	}
2006 
2007 	@property string[string] defaultPostGenerateEnvironments()
2008 	const {
2009 		if (auto pv = "defaultPostGenerateEnvironments" in m_data)
2010 			return deserializeJson!(string[string])(*cast(Json*)pv);
2011 		if (m_parentConfig) return m_parentConfig.defaultPostGenerateEnvironments;
2012 		return null;
2013 	}
2014 
2015 	@property string[string] defaultPreBuildEnvironments()
2016 	const {
2017 		if (auto pv = "defaultPreBuildEnvironments" in m_data)
2018 			return deserializeJson!(string[string])(*cast(Json*)pv);
2019 		if (m_parentConfig) return m_parentConfig.defaultPreBuildEnvironments;
2020 		return null;
2021 	}
2022 
2023 	@property string[string] defaultPostBuildEnvironments()
2024 	const {
2025 		if (auto pv = "defaultPostBuildEnvironments" in m_data)
2026 			return deserializeJson!(string[string])(*cast(Json*)pv);
2027 		if (m_parentConfig) return m_parentConfig.defaultPostBuildEnvironments;
2028 		return null;
2029 	}
2030 
2031 	@property string[string] defaultPreRunEnvironments()
2032 	const {
2033 		if (auto pv = "defaultPreRunEnvironments" in m_data)
2034 			return deserializeJson!(string[string])(*cast(Json*)pv);
2035 		if (m_parentConfig) return m_parentConfig.defaultPreRunEnvironments;
2036 		return null;
2037 	}
2038 
2039 	@property string[string] defaultPostRunEnvironments()
2040 	const {
2041 		if (auto pv = "defaultPostRunEnvironments" in m_data)
2042 			return deserializeJson!(string[string])(*cast(Json*)pv);
2043 		if (m_parentConfig) return m_parentConfig.defaultPostRunEnvironments;
2044 		return null;
2045 	}
2046 }