1 /**
2 	A package manager.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff, 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff, Sönke Ludwig
7 */
8 module dub.dub;
9 
10 import dub.compilers.compiler;
11 import dub.dependency;
12 import dub.dependencyresolver;
13 import dub.internal.utils;
14 import dub.internal.vibecompat.core.file;
15 import dub.internal.vibecompat.core.log;
16 import dub.internal.vibecompat.data.json;
17 import dub.internal.vibecompat.inet.url;
18 import dub.package_;
19 import dub.packagemanager;
20 import dub.packagesuppliers;
21 import dub.project;
22 import dub.generators.generator;
23 import dub.init;
24 
25 import std.algorithm;
26 import std.array : array, replace;
27 import std.conv : to;
28 import std.exception : enforce;
29 import std.file;
30 import std.process : environment;
31 import std.range : assumeSorted, empty;
32 import std.string;
33 import std.encoding : sanitize;
34 
35 // Set output path and options for coverage reports
36 version (DigitalMars) version (D_Coverage)
37 {
38 	shared static this()
39 	{
40 		import core.runtime, std.file, std.path, std.stdio;
41 		dmd_coverSetMerge(true);
42 		auto path = buildPath(dirName(thisExePath()), "../cov");
43 		if (!path.exists)
44 			mkdir(path);
45 		dmd_coverDestPath(path);
46 	}
47 }
48 
49 static this()
50 {
51 	import dub.compilers.dmd : DMDCompiler;
52 	import dub.compilers.gdc : GDCCompiler;
53 	import dub.compilers.ldc : LDCCompiler;
54 	registerCompiler(new DMDCompiler);
55 	registerCompiler(new GDCCompiler);
56 	registerCompiler(new LDCCompiler);
57 }
58 
59 deprecated("use defaultRegistryURLs") enum defaultRegistryURL = defaultRegistryURLs[0];
60 
61 /// The URL to the official package registry and it's default fallback registries.
62 static immutable string[] defaultRegistryURLs = [
63 	"https://code.dlang.org/",
64 	"https://codemirror.dlang.org/",
65 	"https://dub.bytecraft.nl/",
66 	"https://code-mirror.dlang.io/",
67 ];
68 
69 /** Returns a default list of package suppliers.
70 
71 	This will contain a single package supplier that points to the official
72 	package registry.
73 
74 	See_Also: `defaultRegistryURLs`
75 */
76 PackageSupplier[] defaultPackageSuppliers()
77 {
78 	logDiagnostic("Using dub registry url '%s'", defaultRegistryURLs[0]);
79 	return [new FallbackPackageSupplier(defaultRegistryURLs.map!getRegistryPackageSupplier.array)];
80 }
81 
82 /** Returns a registry package supplier according to protocol.
83 
84 	Allowed protocols are dub+http(s):// and maven+http(s)://.
85 */
86 PackageSupplier getRegistryPackageSupplier(string url)
87 {
88 	switch (url.startsWith("dub+", "mvn+", "file://"))
89 	{
90 		case 1:
91 			return new RegistryPackageSupplier(URL(url[4..$]));
92 		case 2:
93 			return new MavenRegistryPackageSupplier(URL(url[4..$]));
94 		case 3:
95 			return new FileSystemPackageSupplier(NativePath(url[7..$]));
96 		default:
97 			return new RegistryPackageSupplier(URL(url));
98 	}
99 }
100 
101 unittest
102 {
103 	auto dubRegistryPackageSupplier = getRegistryPackageSupplier("dub+https://code.dlang.org");
104 	assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org"));
105 
106 	dubRegistryPackageSupplier = getRegistryPackageSupplier("https://code.dlang.org");
107 	assert(dubRegistryPackageSupplier.description.canFind(" https://code.dlang.org"));
108 
109 	auto mavenRegistryPackageSupplier = getRegistryPackageSupplier("mvn+http://localhost:8040/maven/libs-release/dubpackages");
110 	assert(mavenRegistryPackageSupplier.description.canFind(" http://localhost:8040/maven/libs-release/dubpackages"));
111 
112 	auto fileSystemPackageSupplier = getRegistryPackageSupplier("file:///etc/dubpackages");
113 	assert(fileSystemPackageSupplier.description.canFind(" " ~ NativePath("/etc/dubpackages").toNativeString));
114 }
115 
116 /** Provides a high-level entry point for DUB's functionality.
117 
118 	This class provides means to load a certain project (a root package with
119 	all of its dependencies) and to perform high-level operations as found in
120 	the command line interface.
121 */
122 class Dub {
123 	private {
124 		bool m_dryRun = false;
125 		PackageManager m_packageManager;
126 		PackageSupplier[] m_packageSuppliers;
127 		NativePath m_rootPath;
128 		SpecialDirs m_dirs;
129 		DubConfig m_config;
130 		NativePath m_projectPath;
131 		Project m_project;
132 		NativePath m_overrideSearchPath;
133 		string m_defaultCompiler;
134 		string m_defaultArchitecture;
135 		bool m_defaultLowMemory;
136 		string[string] m_defaultEnvironments;
137 		string[string] m_defaultBuildEnvironments;
138 		string[string] m_defaultRunEnvironments;
139 		string[string] m_defaultPreGenerateEnvironments;
140 		string[string] m_defaultPostGenerateEnvironments;
141 		string[string] m_defaultPreBuildEnvironments;
142 		string[string] m_defaultPostBuildEnvironments;
143 		string[string] m_defaultPreRunEnvironments;
144 		string[string] m_defaultPostRunEnvironments;
145 
146 	}
147 
148 	/** The default placement location of fetched packages.
149 
150 		This property can be altered, so that packages which are downloaded as part
151 		of the normal upgrade process are stored in a certain location. This is
152 		how the "--local" and "--system" command line switches operate.
153 	*/
154 	PlacementLocation defaultPlacementLocation = PlacementLocation.user;
155 
156 
157 	/** Initializes the instance for use with a specific root package.
158 
159 		Note that a package still has to be loaded using one of the
160 		`loadPackage` overloads.
161 
162 		Params:
163 			root_path = Path to the root package
164 			additional_package_suppliers = A list of package suppliers to try
165 				before the suppliers found in the configurations files and the
166 				`defaultPackageSuppliers`.
167 			skip_registry = Can be used to skip using the configured package
168 				suppliers, as well as the default suppliers.
169 	*/
170 	this(string root_path = ".", PackageSupplier[] additional_package_suppliers = null,
171 			SkipPackageSuppliers skip_registry = SkipPackageSuppliers.none)
172 	{
173 		m_rootPath = NativePath(root_path);
174 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
175 
176 		init(m_rootPath);
177 
178 		m_packageSuppliers = getPackageSuppliers(additional_package_suppliers, skip_registry);
179 		m_packageManager = new PackageManager(m_rootPath, m_dirs.localRepository, m_dirs.systemSettings);
180 
181 		auto ccps = m_config.customCachePaths;
182 		if (ccps.length)
183 			m_packageManager.customCachePaths = ccps;
184 
185 		updatePackageSearchPath();
186 	}
187 
188 	unittest
189 	{
190 		scope (exit) environment.remove("DUB_REGISTRY");
191 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
192 		assert(dub.m_packageSuppliers.length == 0);
193 		environment["DUB_REGISTRY"] = "http://example.com/";
194 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
195 		assert(dub.m_packageSuppliers.length == 1);
196 		environment["DUB_REGISTRY"] = "http://example.com/;http://foo.com/";
197 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
198 		assert(dub.m_packageSuppliers.length == 2);
199 		dub = new Dub(".", [new RegistryPackageSupplier(URL("http://bar.com/"))], SkipPackageSuppliers.configured);
200 		assert(dub.m_packageSuppliers.length == 3);
201 	}
202 
203 	/** Get the list of package suppliers.
204 
205 		Params:
206 			additional_package_suppliers = A list of package suppliers to try
207 				before the suppliers found in the configurations files and the
208 				`defaultPackageSuppliers`.
209 			skip_registry = Can be used to skip using the configured package
210 				suppliers, as well as the default suppliers.
211 	*/
212 	public PackageSupplier[] getPackageSuppliers(PackageSupplier[] additional_package_suppliers, SkipPackageSuppliers skip_registry)
213 	{
214 		PackageSupplier[] ps = additional_package_suppliers;
215 
216 		if (skip_registry < SkipPackageSuppliers.all)
217 		{
218 			ps ~= environment.get("DUB_REGISTRY", null)
219 				.splitter(";")
220 				.map!(url => getRegistryPackageSupplier(url))
221 				.array;
222 		}
223 
224 		if (skip_registry < SkipPackageSuppliers.configured)
225 		{
226 			ps ~= m_config.registryURLs
227 				.map!(url => getRegistryPackageSupplier(url))
228 				.array;
229 		}
230 
231 		if (skip_registry < SkipPackageSuppliers.standard)
232 			ps ~= defaultPackageSuppliers();
233 
234 		return ps;
235 	}
236 
237 	/// ditto
238 	public PackageSupplier[] getPackageSuppliers(PackageSupplier[] additional_package_suppliers)
239 	{
240 		return getPackageSuppliers(additional_package_suppliers, m_config.skipRegistry);
241 	}
242 
243 	unittest
244 	{
245 		scope (exit) environment.remove("DUB_REGISTRY");
246 		auto dub = new Dub(".", null, SkipPackageSuppliers.none);
247 
248 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("none")]), null);
249 		assert(dub.getPackageSuppliers(null).length == 1);
250 
251 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("configured")]), null);
252 		assert(dub.getPackageSuppliers(null).length == 0);
253 
254 		dub.m_config = new DubConfig(Json(["skipRegistry": Json("standard")]), null);
255 		assert(dub.getPackageSuppliers(null).length == 0);
256 
257 		environment["DUB_REGISTRY"] = "http://example.com/";
258 		assert(dub.getPackageSuppliers(null).length == 1);
259 	}
260 
261 	/** Initializes the instance with a single package search path, without
262 		loading a package.
263 
264 		This constructor corresponds to the "--bare" option of the command line
265 		interface. Use
266 	*/
267 	this(NativePath override_path)
268 	{
269 		init(NativePath());
270 		m_overrideSearchPath = override_path;
271 		m_packageManager = new PackageManager(override_path);
272 	}
273 
274 	private void init(NativePath root_path)
275 	{
276 		import std.file : tempDir;
277 		version(Windows) {
278 			m_dirs.systemSettings = NativePath(environment.get("ProgramData")) ~ "dub/";
279 			immutable appDataDir = environment.get("APPDATA");
280 			m_dirs.userSettings = NativePath(appDataDir) ~ "dub/";
281 			m_dirs.localRepository = NativePath(environment.get("LOCALAPPDATA", appDataDir)) ~ "dub";
282 		} else version(Posix){
283 			m_dirs.systemSettings = NativePath("/var/lib/dub/");
284 			m_dirs.userSettings = NativePath(environment.get("HOME")) ~ ".dub/";
285 			if (!m_dirs.userSettings.absolute)
286 				m_dirs.userSettings = NativePath(getcwd()) ~ m_dirs.userSettings;
287 			m_dirs.localRepository = m_dirs.userSettings;
288 		}
289 
290 		m_dirs.temp = NativePath(tempDir);
291 
292 		m_config = new DubConfig(jsonFromFile(m_dirs.systemSettings ~ "settings.json", true), m_config);
293 
294 		auto dubFolderPath = NativePath(thisExePath).parentPath;
295 		m_config = new DubConfig(jsonFromFile(dubFolderPath ~ "../etc/dub/settings.json", true), m_config);
296 		version (Posix) {
297 			if (dubFolderPath.absolute && dubFolderPath.startsWith(NativePath("usr"))) {
298 				m_config = new DubConfig(jsonFromFile(NativePath("/etc/dub/settings.json"), true), m_config);
299 			}
300 		}
301 
302 		m_config = new DubConfig(jsonFromFile(m_dirs.userSettings ~ "settings.json", true), m_config);
303 
304 		if (!root_path.empty)
305 			m_config = new DubConfig(jsonFromFile(root_path ~ "dub.settings.json", true), m_config);
306 
307 		determineDefaultCompiler();
308 
309 		m_defaultArchitecture = m_config.defaultArchitecture;
310 		m_defaultLowMemory = m_config.defaultLowMemory;
311 		m_defaultEnvironments = m_config.defaultEnvironments;
312 		m_defaultBuildEnvironments = m_config.defaultBuildEnvironments;
313 		m_defaultRunEnvironments = m_config.defaultRunEnvironments;
314 		m_defaultPreGenerateEnvironments = m_config.defaultPreGenerateEnvironments;
315 		m_defaultPostGenerateEnvironments = m_config.defaultPostGenerateEnvironments;
316 		m_defaultPreBuildEnvironments = m_config.defaultPreBuildEnvironments;
317 		m_defaultPostBuildEnvironments = m_config.defaultPostBuildEnvironments;
318 		m_defaultPreRunEnvironments = m_config.defaultPreRunEnvironments;
319 		m_defaultPostRunEnvironments = m_config.defaultPostRunEnvironments;
320 	}
321 
322 	@property void dryRun(bool v) { m_dryRun = v; }
323 
324 	/** Returns the root path (usually the current working directory).
325 	*/
326 	@property NativePath rootPath() const { return m_rootPath; }
327 	/// ditto
328 	@property void rootPath(NativePath root_path)
329 	{
330 		m_rootPath = root_path;
331 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
332 	}
333 
334 	/// Returns the name listed in the dub.json of the current
335 	/// application.
336 	@property string projectName() const { return m_project.name; }
337 
338 	@property NativePath projectPath() const { return m_projectPath; }
339 
340 	@property string[] configurations() const { return m_project.configurations; }
341 
342 	@property inout(PackageManager) packageManager() inout { return m_packageManager; }
343 
344 	@property inout(Project) project() inout { return m_project; }
345 
346 	/** Returns the default compiler binary to use for building D code.
347 
348 		If set, the "defaultCompiler" field of the DUB user or system
349 		configuration file will be used. Otherwise the PATH environment variable
350 		will be searched for files named "dmd", "gdc", "gdmd", "ldc2", "ldmd2"
351 		(in that order, taking into account operating system specific file
352 		extensions) and the first match is returned. If no match is found, "dmd"
353 		will be used.
354 	*/
355 	@property string defaultCompiler() const { return m_defaultCompiler; }
356 
357 	/** Returns the default architecture to use for building D code.
358 
359 		If set, the "defaultArchitecture" field of the DUB user or system
360 		configuration file will be used. Otherwise null will be returned.
361 	*/
362 	@property string defaultArchitecture() const { return m_defaultArchitecture; }
363 
364 	/** Returns the default low memory option to use for building D code.
365 
366 		If set, the "defaultLowMemory" field of the DUB user or system
367 		configuration file will be used. Otherwise false will be returned.
368 	*/
369 	@property bool defaultLowMemory() const { return m_defaultLowMemory; }
370 
371 	@property const(string[string]) defaultEnvironments() const { return m_defaultEnvironments; }
372 	@property const(string[string]) defaultBuildEnvironments() const { return m_defaultBuildEnvironments; }
373 	@property const(string[string]) defaultRunEnvironments() const { return m_defaultRunEnvironments; }
374 	@property const(string[string]) defaultPreGenerateEnvironments() const { return m_defaultPreGenerateEnvironments; }
375 	@property const(string[string]) defaultPostGenerateEnvironments() const { return m_defaultPostGenerateEnvironments; }
376 	@property const(string[string]) defaultPreBuildEnvironments() const { return m_defaultPreBuildEnvironments; }
377 	@property const(string[string]) defaultPostBuildEnvironments() const { return m_defaultPostBuildEnvironments; }
378 	@property const(string[string]) defaultPreRunEnvironments() const { return m_defaultPreRunEnvironments; }
379 	@property const(string[string]) defaultPostRunEnvironments() const { return m_defaultPostRunEnvironments; }
380 
381 	/** Loads the package that resides within the configured `rootPath`.
382 	*/
383 	void loadPackage()
384 	{
385 		loadPackage(m_rootPath);
386 	}
387 
388 	/// Loads the package from the specified path as the main project package.
389 	void loadPackage(NativePath path)
390 	{
391 		m_projectPath = path;
392 		updatePackageSearchPath();
393 		m_project = new Project(m_packageManager, m_projectPath);
394 	}
395 
396 	/// Loads a specific package as the main project package (can be a sub package)
397 	void loadPackage(Package pack)
398 	{
399 		m_projectPath = pack.path;
400 		updatePackageSearchPath();
401 		m_project = new Project(m_packageManager, pack);
402 	}
403 
404 	/** Loads a single file package.
405 
406 		Single-file packages are D files that contain a package receipe comment
407 		at their top. A recipe comment must be a nested `/+ ... +/` style
408 		comment, containing the virtual recipe file name and a colon, followed by the
409 		recipe contents (what would normally be in dub.sdl/dub.json).
410 
411 		Example:
412 		---
413 		/+ dub.sdl:
414 		   name "test"
415 		   dependency "vibe-d" version="~>0.7.29"
416 		+/
417 		import vibe.http.server;
418 
419 		void main()
420 		{
421 			auto settings = new HTTPServerSettings;
422 			settings.port = 8080;
423 			listenHTTP(settings, &hello);
424 		}
425 
426 		void hello(HTTPServerRequest req, HTTPServerResponse res)
427 		{
428 			res.writeBody("Hello, World!");
429 		}
430 		---
431 
432 		The script above can be invoked with "dub --single test.d".
433 	*/
434 	void loadSingleFilePackage(NativePath path)
435 	{
436 		import dub.recipe.io : parsePackageRecipe;
437 		import std.file : mkdirRecurse, readText;
438 		import std.path : baseName, stripExtension;
439 
440 		path = makeAbsolute(path);
441 
442 		string file_content = readText(path.toNativeString());
443 
444 		if (file_content.startsWith("#!")) {
445 			auto idx = file_content.indexOf('\n');
446 			enforce(idx > 0, "The source fine doesn't contain anything but a shebang line.");
447 			file_content = file_content[idx+1 .. $];
448 		}
449 
450 		file_content = file_content.strip();
451 
452 		string recipe_content;
453 
454 		if (file_content.startsWith("/+")) {
455 			file_content = file_content[2 .. $];
456 			auto idx = file_content.indexOf("+/");
457 			enforce(idx >= 0, "Missing \"+/\" to close comment.");
458 			recipe_content = file_content[0 .. idx].strip();
459 		} else throw new Exception("The source file must start with a recipe comment.");
460 
461 		auto nidx = recipe_content.indexOf('\n');
462 
463 		auto idx = recipe_content.indexOf(':');
464 		enforce(idx > 0 && (nidx < 0 || nidx > idx),
465 			"The first line of the recipe comment must list the recipe file name followed by a colon (e.g. \"/+ dub.sdl:\").");
466 		auto recipe_filename = recipe_content[0 .. idx];
467 		recipe_content = recipe_content[idx+1 .. $];
468 		auto recipe_default_package_name = path.toString.baseName.stripExtension.strip;
469 
470 		auto recipe = parsePackageRecipe(recipe_content, recipe_filename, null, recipe_default_package_name);
471 		import dub.internal.vibecompat.core.log; logInfo("parsePackageRecipe %s", recipe_filename);
472 		enforce(recipe.buildSettings.sourceFiles.length == 0, "Single-file packages are not allowed to specify source files.");
473 		enforce(recipe.buildSettings.sourcePaths.length == 0, "Single-file packages are not allowed to specify source paths.");
474 		enforce(recipe.buildSettings.importPaths.length == 0, "Single-file packages are not allowed to specify import paths.");
475 		recipe.buildSettings.sourceFiles[""] = [path.toNativeString()];
476 		recipe.buildSettings.sourcePaths[""] = [];
477 		recipe.buildSettings.importPaths[""] = [];
478 		recipe.buildSettings.mainSourceFile = path.toNativeString();
479 		if (recipe.buildSettings.targetType == TargetType.autodetect)
480 			recipe.buildSettings.targetType = TargetType.executable;
481 
482 		auto pack = new Package(recipe, path.parentPath, null, "~master");
483 		loadPackage(pack);
484 	}
485 	/// ditto
486 	void loadSingleFilePackage(string path)
487 	{
488 		loadSingleFilePackage(NativePath(path));
489 	}
490 
491 	deprecated("Instantiate a Dub instance with the single-argument constructor: `new Dub(path)`")
492 	void overrideSearchPath(NativePath path)
493 	{
494 		if (!path.absolute) path = NativePath(getcwd()) ~ path;
495 		m_overrideSearchPath = path;
496 		updatePackageSearchPath();
497 	}
498 
499 	/** Gets the default configuration for a particular build platform.
500 
501 		This forwards to `Project.getDefaultConfiguration` and requires a
502 		project to be loaded.
503 	*/
504 	string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); }
505 
506 	/** Attempts to upgrade the dependency selection of the loaded project.
507 
508 		Params:
509 			options = Flags that control how the upgrade is carried out
510 			packages_to_upgrade = Optional list of packages. If this list
511 				contains one or more packages, only those packages will
512 				be upgraded. Otherwise, all packages will be upgraded at
513 				once.
514 	*/
515 	void upgrade(UpgradeOptions options, string[] packages_to_upgrade = null)
516 	{
517 		// clear non-existent version selections
518 		if (!(options & UpgradeOptions.upgrade)) {
519 			next_pack:
520 			foreach (p; m_project.selections.selectedPackages) {
521 				auto dep = m_project.selections.getSelectedVersion(p);
522 				if (!dep.path.empty) {
523 					auto path = dep.path;
524 					if (!path.absolute) path = this.rootPath ~ path;
525 					try if (m_packageManager.getOrLoadPackage(path)) continue;
526 					catch (Exception e) { logDebug("Failed to load path based selection: %s", e.toString().sanitize); }
527 				} else if (!dep.repository.empty) {
528 					if (m_packageManager.loadSCMPackage(getBasePackageName(p), dep))
529 						continue;
530 				} else {
531 					if (m_packageManager.getPackage(p, dep.version_)) continue;
532 					foreach (ps; m_packageSuppliers) {
533 						try {
534 							auto versions = ps.getVersions(p);
535 							if (versions.canFind!(v => dep.matches(v)))
536 								continue next_pack;
537 						} catch (Exception e) {
538 							logWarn("Error querying versions for %s, %s: %s", p, ps.description, e.msg);
539 							logDebug("Full error: %s", e.toString().sanitize());
540 						}
541 					}
542 				}
543 
544 				logWarn("Selected package %s %s doesn't exist. Using latest matching version instead.", p, dep);
545 				m_project.selections.deselectVersion(p);
546 			}
547 		}
548 
549 		Dependency[string] versions;
550 		auto resolver = new DependencyVersionResolver(this, options);
551 		foreach (p; packages_to_upgrade)
552 			resolver.addPackageToUpgrade(p);
553 		versions = resolver.resolve(m_project.rootPackage, m_project.selections);
554 
555 		if (options & UpgradeOptions.dryRun) {
556 			bool any = false;
557 			string rootbasename = getBasePackageName(m_project.rootPackage.name);
558 
559 			foreach (p, ver; versions) {
560 				if (!ver.path.empty || !ver.repository.empty) continue;
561 
562 				auto basename = getBasePackageName(p);
563 				if (basename == rootbasename) continue;
564 
565 				if (!m_project.selections.hasSelectedVersion(basename)) {
566 					logInfo("Package %s would be selected with version %s.",
567 						basename, ver);
568 					any = true;
569 					continue;
570 				}
571 				auto sver = m_project.selections.getSelectedVersion(basename);
572 				if (!sver.path.empty || !sver.repository.empty) continue;
573 				if (ver.version_ <= sver.version_) continue;
574 				logInfo("Package %s would be upgraded from %s to %s.",
575 					basename, sver, ver);
576 				any = true;
577 			}
578 			if (any) logInfo("Use \"dub upgrade\" to perform those changes.");
579 			return;
580 		}
581 
582 		foreach (p; versions.byKey) {
583 			auto ver = versions[p]; // Workaround for DMD 2.070.0 AA issue (crashes in aaApply2 if iterating by key+value)
584 			assert(!p.canFind(":"), "Resolved packages contain a sub package!?: "~p);
585 			Package pack;
586 			if (!ver.path.empty) {
587 				try pack = m_packageManager.getOrLoadPackage(ver.path);
588 				catch (Exception e) {
589 					logDebug("Failed to load path based selection: %s", e.toString().sanitize);
590 					continue;
591 				}
592 			} else if (!ver.repository.empty) {
593 				pack = m_packageManager.loadSCMPackage(p, ver);
594 			} else {
595 				pack = m_packageManager.getBestPackage(p, ver);
596 				if (pack && m_packageManager.isManagedPackage(pack)
597 					&& ver.version_.isBranch && (options & UpgradeOptions.upgrade) != 0)
598 				{
599 					// TODO: only re-install if there is actually a new commit available
600 					logInfo("Re-installing branch based dependency %s %s", p, ver.toString());
601 					m_packageManager.remove(pack);
602 					pack = null;
603 				}
604 			}
605 
606 			FetchOptions fetchOpts;
607 			fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none;
608 			if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version");
609 			if ((options & UpgradeOptions.select) && p != m_project.rootPackage.name) {
610 				if (!ver.repository.empty) {
611 					m_project.selections.selectVersionWithRepository(p, ver.repository, ver.versionSpec);
612 				} else if (ver.path.empty) {
613 					m_project.selections.selectVersion(p, ver.version_);
614 				} else {
615 					NativePath relpath = ver.path;
616 					if (relpath.absolute) relpath = relpath.relativeTo(m_project.rootPackage.path);
617 					m_project.selections.selectVersion(p, relpath);
618 				}
619 			}
620 		}
621 
622 		string[] missingDependenciesBeforeReinit = m_project.missingDependencies;
623 		m_project.reinit();
624 
625 		if (!m_project.hasAllDependencies) {
626 			auto resolvedDependencies = setDifference(
627 					assumeSorted(missingDependenciesBeforeReinit),
628 					assumeSorted(m_project.missingDependencies)
629 				);
630 			if (!resolvedDependencies.empty)
631 				upgrade(options, m_project.missingDependencies);
632 		}
633 
634 		if ((options & UpgradeOptions.select) && !(options & (UpgradeOptions.noSaveSelections | UpgradeOptions.dryRun)))
635 			m_project.saveSelections();
636 	}
637 
638 	/** Generate project files for a specified generator.
639 
640 		Any existing project files will be overridden.
641 	*/
642 	void generateProject(string ide, GeneratorSettings settings)
643 	{
644 		auto generator = createProjectGenerator(ide, m_project);
645 		if (m_dryRun) return; // TODO: pass m_dryRun to the generator
646 		generator.generate(settings);
647 	}
648 
649 	/** Executes tests on the current project.
650 
651 		Throws an exception, if unittests failed.
652 	*/
653 	void testProject(GeneratorSettings settings, string config, NativePath custom_main_file)
654 	{
655 		if (!custom_main_file.empty && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file;
656 
657 		if (config.length == 0) {
658 			// if a custom main file was given, favor the first library configuration, so that it can be applied
659 			if (!custom_main_file.empty) config = m_project.getDefaultConfiguration(settings.platform, false);
660 			// else look for a "unittest" configuration
661 			if (!config.length && m_project.rootPackage.configurations.canFind("unittest")) config = "unittest";
662 			// if not found, fall back to the first "library" configuration
663 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, false);
664 			// if still nothing found, use the first executable configuration
665 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, true);
666 		}
667 
668 		auto generator = createProjectGenerator("build", m_project);
669 
670 		auto test_config = format("%s-test-%s", m_project.rootPackage.name.replace(".", "-").replace(":", "-"), config);
671 
672 		BuildSettings lbuildsettings = settings.buildSettings;
673 		m_project.addBuildSettings(lbuildsettings, settings, config, null, true);
674 
675 		if (lbuildsettings.targetType == TargetType.none) {
676 			logInfo(`Configuration '%s' has target type "none". Skipping test.`, config);
677 			return;
678 		}
679 
680 		if (lbuildsettings.targetType == TargetType.executable && config == "unittest") {
681 			logInfo("Running custom 'unittest' configuration.", config);
682 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
683 			settings.config = config;
684 		} else if (lbuildsettings.sourceFiles.empty) {
685 			logInfo(`No source files found in configuration '%s'. Falling back to "dub -b unittest".`, config);
686 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
687 			settings.config = m_project.getDefaultConfiguration(settings.platform);
688 		} else {
689 			import std.algorithm : remove;
690 
691 			logInfo(`Generating test runner configuration '%s' for '%s' (%s).`, test_config, config, lbuildsettings.targetType);
692 
693 			BuildSettingsTemplate tcinfo = m_project.rootPackage.recipe.getConfiguration(config).buildSettings;
694 			tcinfo.targetType = TargetType.executable;
695 			tcinfo.targetName = test_config;
696 
697 			auto mainfil = tcinfo.mainSourceFile;
698 			if (!mainfil.length) mainfil = m_project.rootPackage.recipe.buildSettings.mainSourceFile;
699 
700 			string custommodname;
701 			if (!custom_main_file.empty) {
702 				import std.path;
703 				tcinfo.sourceFiles[""] ~= custom_main_file.relativeTo(m_project.rootPackage.path).toNativeString();
704 				tcinfo.importPaths[""] ~= custom_main_file.parentPath.toNativeString();
705 				custommodname = custom_main_file.head.name.baseName(".d");
706 			}
707 
708 			// prepare the list of tested modules
709 
710 			string[] import_modules;
711 			if (settings.single)
712 				lbuildsettings.importPaths ~= NativePath(mainfil).parentPath.toNativeString;
713 			bool firstTimePackage = true;
714 			foreach (file; lbuildsettings.sourceFiles) {
715 				if (file.endsWith(".d")) {
716 					auto fname = NativePath(file).head.name;
717 					NativePath msf = NativePath(mainfil);
718 					if (msf.absolute)
719 						msf = msf.relativeTo(m_project.rootPackage.path);
720 					if (!settings.single && NativePath(file).relativeTo(m_project.rootPackage.path) == msf) {
721 						logWarn("Excluding main source file %s from test.", mainfil);
722 						tcinfo.excludedSourceFiles[""] ~= mainfil;
723 						continue;
724 					}
725 					if (fname == "package.d") {
726 						if (firstTimePackage) {
727 							firstTimePackage = false;
728 							logDiagnostic("Excluding package.d file from test due to https://issues.dlang.org/show_bug.cgi?id=11847");
729 						}
730 						continue;
731 					}
732 					import_modules ~= dub.internal.utils.determineModuleName(lbuildsettings, NativePath(file), m_project.rootPackage.path);
733 				}
734 			}
735 
736 			NativePath mainfile;
737 			if (settings.tempBuild)
738 				mainfile = getTempFile("dub_test_root", ".d");
739 			else {
740 				import dub.generators.build : computeBuildName;
741 				mainfile = m_project.rootPackage.path ~ format(".dub/code/%s_dub_test_root.d", computeBuildName(test_config, settings, import_modules));
742 			}
743 
744 			mkdirRecurse(mainfile.parentPath.toNativeString());
745 
746 			bool regenerateMainFile = settings.force || !existsFile(mainfile);
747 			auto escapedMainFile = mainfile.toNativeString().replace("$", "$$");
748 			// generate main file
749 			tcinfo.sourceFiles[""] ~= escapedMainFile;
750 			tcinfo.mainSourceFile = escapedMainFile;
751 
752 			if (!m_dryRun && regenerateMainFile) {
753 				auto fil = openFile(mainfile, FileMode.createTrunc);
754 				scope(exit) fil.close();
755 				fil.write("module dub_test_root;\n");
756 				fil.write("import std.typetuple;\n");
757 				foreach (mod; import_modules) fil.write(format("static import %s;\n", mod));
758 				fil.write("alias allModules = TypeTuple!(");
759 				foreach (i, mod; import_modules) {
760 					if (i > 0) fil.write(", ");
761 					fil.write(mod);
762 				}
763 				fil.write(");\n");
764 				if (custommodname.length) {
765 					fil.write(format("import %s;\n", custommodname));
766 				} else {
767 					fil.write(q{
768 						import std.stdio;
769 						import core.runtime;
770 
771 						void main() { writeln("All unit tests have been run successfully."); }
772 						shared static this() {
773 							version (Have_tested) {
774 								import tested;
775 								import core.runtime;
776 								import std.exception;
777 								Runtime.moduleUnitTester = () => true;
778 								enforce(runUnitTests!allModules(new ConsoleTestResultWriter), "Unit tests failed.");
779 							}
780 						}
781 					});
782 				}
783 			}
784 			m_project.rootPackage.recipe.configurations ~= ConfigurationInfo(test_config, tcinfo);
785 			m_project = new Project(m_packageManager, m_project.rootPackage);
786 
787 			settings.config = test_config;
788 		}
789 
790 		generator.generate(settings);
791 	}
792 
793 	/** Executes D-Scanner tests on the current project. **/
794 	void lintProject(string[] args)
795 	{
796 		import std.path : buildPath, buildNormalizedPath;
797 
798 		if (m_dryRun) return;
799 
800 		auto tool = "dscanner";
801 
802 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
803 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
804 		if (!tool_pack) {
805 			logInfo("%s is not present, getting and storing it user wide", tool);
806 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
807 		}
808 
809 		auto dscanner_dub = new Dub(null, m_packageSuppliers);
810 		dscanner_dub.loadPackage(tool_pack.path);
811 		dscanner_dub.upgrade(UpgradeOptions.select);
812 
813 		auto compiler_binary = this.defaultCompiler;
814 
815 		GeneratorSettings settings;
816 		settings.config = "application";
817 		settings.compiler = getCompiler(compiler_binary);
818 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
819 		settings.buildType = "debug";
820 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
821 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
822 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
823 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
824 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
825 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
826 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
827 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
828 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
829 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
830 		settings.run = true;
831 
832 		foreach (dependencyPackage; m_project.dependencies)
833 		{
834 			auto cfgs = m_project.getPackageConfigs(settings.platform, null, true);
835 			auto buildSettings = dependencyPackage.getBuildSettings(settings.platform, cfgs[dependencyPackage.name]);
836 			foreach (importPath; buildSettings.importPaths) {
837 				settings.runArgs ~= ["-I", buildNormalizedPath(dependencyPackage.path.toNativeString(), importPath.idup)];
838 			}
839 		}
840 
841 		string configFilePath = buildPath(m_project.rootPackage.path.toNativeString(), "dscanner.ini");
842 		if (!args.canFind("--config") && exists(configFilePath)) {
843 			settings.runArgs ~= ["--config", configFilePath];
844 		}
845 
846 		settings.runArgs ~= args ~ [m_project.rootPackage.path.toNativeString()];
847 		dscanner_dub.generateProject("build", settings);
848 	}
849 
850 	/** Prints the specified build settings necessary for building the root package.
851 	*/
852 	void listProjectData(GeneratorSettings settings, string[] requestedData, ListBuildSettingsFormat list_type)
853 	{
854 		import std.stdio;
855 		import std.ascii : newline;
856 
857 		// Split comma-separated lists
858 		string[] requestedDataSplit =
859 			requestedData
860 			.map!(a => a.splitter(",").map!strip)
861 			.joiner()
862 			.array();
863 
864 		auto data = m_project.listBuildSettings(settings, requestedDataSplit, list_type);
865 
866 		string delimiter;
867 		final switch (list_type) with (ListBuildSettingsFormat) {
868 			case list: delimiter = newline ~ newline; break;
869 			case listNul: delimiter = "\0\0"; break;
870 			case commandLine: delimiter = " "; break;
871 			case commandLineNul: delimiter = "\0\0"; break;
872 		}
873 
874 		write(data.joiner(delimiter));
875 		if (delimiter != "\0\0") writeln();
876 	}
877 
878 	/// Cleans intermediate/cache files of the given package
879 	void cleanPackage(NativePath path)
880 	{
881 		logInfo("Cleaning package at %s...", path.toNativeString());
882 		enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString());
883 
884 		// TODO: clear target files and copy files
885 
886 		if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString());
887 		if (existsFile(path ~ ".dub/metadata_cache.json")) std.file.remove((path ~ ".dub/metadata_cache.json").toNativeString());
888 
889 		auto p = Package.load(path);
890 		if (p.getBuildSettings().targetType == TargetType.none) {
891 			foreach (sp; p.subPackages.filter!(sp => !sp.path.empty)) {
892 				cleanPackage(path ~ sp.path);
893 			}
894 		}
895 	}
896 
897 	/// Fetches the package matching the dependency and places it in the specified location.
898 	Package fetch(string packageId, const Dependency dep, PlacementLocation location, FetchOptions options, string reason = "")
899 	{
900 		auto basePackageName = getBasePackageName(packageId);
901 		Json pinfo;
902 		PackageSupplier supplier;
903 		foreach(ps; m_packageSuppliers){
904 			try {
905 				pinfo = ps.fetchPackageRecipe(basePackageName, dep, (options & FetchOptions.usePrerelease) != 0);
906 				if (pinfo.type == Json.Type.null_)
907 					continue;
908 				supplier = ps;
909 				break;
910 			} catch(Exception e) {
911 				logWarn("Package %s not found for %s: %s", packageId, ps.description, e.msg);
912 				logDebug("Full error: %s", e.toString().sanitize());
913 			}
914 		}
915 		enforce(pinfo.type != Json.Type.undefined, "No package "~packageId~" was found matching the dependency "~dep.toString());
916 		string ver = pinfo["version"].get!string;
917 
918 		NativePath placement;
919 		final switch (location) {
920 			case PlacementLocation.local: placement = m_rootPath ~ ".dub/packages/"; break;
921 			case PlacementLocation.user: placement = m_dirs.localRepository ~ "packages/"; break;
922 			case PlacementLocation.system: placement = m_dirs.systemSettings ~ "packages/"; break;
923 		}
924 
925 		// always upgrade branch based versions - TODO: actually check if there is a new commit available
926 		Package existing;
927 		try existing = m_packageManager.getPackage(packageId, ver, placement);
928 		catch (Exception e) {
929 			logWarn("Failed to load existing package %s: %s", ver, e.msg);
930 			logDiagnostic("Full error: %s", e.toString().sanitize);
931 		}
932 
933 		if (options & FetchOptions.printOnly) {
934 			if (existing && existing.version_ != Version(ver))
935 				logInfo("A new version for %s is available (%s -> %s). Run \"dub upgrade %s\" to switch.",
936 					packageId, existing.version_, ver, packageId);
937 			return null;
938 		}
939 
940 		if (existing) {
941 			if (!ver.startsWith("~") || !(options & FetchOptions.forceBranchUpgrade) || location == PlacementLocation.local) {
942 				// TODO: support git working trees by performing a "git pull" instead of this
943 				logDiagnostic("Package %s %s (%s) is already present with the latest version, skipping upgrade.",
944 					packageId, ver, placement);
945 				return existing;
946 			} else {
947 				logInfo("Removing %s %s to prepare replacement with a new version.", packageId, ver);
948 				if (!m_dryRun) m_packageManager.remove(existing);
949 			}
950 		}
951 
952 		if (reason.length) logInfo("Fetching %s %s (%s)...", packageId, ver, reason);
953 		else logInfo("Fetching %s %s...", packageId, ver);
954 		if (m_dryRun) return null;
955 
956 		logDebug("Acquiring package zip file");
957 
958 		auto clean_package_version = ver[ver.startsWith("~") ? 1 : 0 .. $];
959 		clean_package_version = clean_package_version.replace("+", "_"); // + has special meaning for Optlink
960 		if (!placement.existsFile())
961 			mkdirRecurse(placement.toNativeString());
962 		NativePath dstpath = placement ~ (basePackageName ~ "-" ~ clean_package_version);
963 		if (!dstpath.existsFile())
964 			mkdirRecurse(dstpath.toNativeString());
965 
966 		// Support libraries typically used with git submodules like ae.
967 		// Such libraries need to have ".." as import path but this can create
968 		// import path leakage.
969 		dstpath = dstpath ~ basePackageName;
970 
971 		import std.datetime : seconds;
972 		auto lock = lockFile(dstpath.toNativeString() ~ ".lock", 30.seconds); // possibly wait for other dub instance
973 		if (dstpath.existsFile())
974 		{
975 			m_packageManager.refresh(false);
976 			return m_packageManager.getPackage(packageId, ver, dstpath);
977 		}
978 
979 		// repeat download on corrupted zips, see #1336
980 		foreach_reverse (i; 0..3)
981 		{
982 			import std.zip : ZipException;
983 
984 			auto path = getTempFile(basePackageName, ".zip");
985 			supplier.fetchPackage(path, basePackageName, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
986 			scope(exit) std.file.remove(path.toNativeString());
987 			logDiagnostic("Placing to %s...", placement.toNativeString());
988 
989 			try {
990 				m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
991 				return m_packageManager.getPackage(packageId, ver, dstpath);
992 			} catch (ZipException e) {
993 				logInfo("Failed to extract zip archive for %s %s...", packageId, ver);
994 				// rethrow the exception at the end of the loop
995 				if (i == 0)
996 					throw e;
997 			}
998 		}
999 		assert(0, "Should throw a ZipException instead.");
1000 	}
1001 
1002 	/** Removes a specific locally cached package.
1003 
1004 		This will delete the package files from disk and removes the
1005 		corresponding entry from the list of known packages.
1006 
1007 		Params:
1008 			pack = Package instance to remove
1009 	*/
1010 	void remove(in Package pack)
1011 	{
1012 		logInfo("Removing %s in %s", pack.name, pack.path.toNativeString());
1013 		if (!m_dryRun) m_packageManager.remove(pack);
1014 	}
1015 
1016 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1017 	void remove(in Package pack, bool force_remove)
1018 	{
1019 		remove(pack);
1020 	}
1021 
1022 	/// @see remove(string, string, RemoveLocation)
1023 	enum RemoveVersionWildcard = "*";
1024 
1025 	/** Removes one or more versions of a locally cached package.
1026 
1027 		This will remove a given package with a specified version from the
1028 		given location. It will remove at most one package, unless `version_`
1029 		is set to `RemoveVersionWildcard`.
1030 
1031 		Params:
1032 			package_id = Name of the package to be removed
1033 			location_ = Specifies the location to look for the given package
1034 				name/version.
1035 			resolve_version = Callback to select package version.
1036 	*/
1037 	void remove(string package_id, PlacementLocation location,
1038 				scope size_t delegate(in Package[] packages) resolve_version)
1039 	{
1040 		enforce(!package_id.empty);
1041 		if (location == PlacementLocation.local) {
1042 			logInfo("To remove a locally placed package, make sure you don't have any data"
1043 					~ "\nleft in it's directory and then simply remove the whole directory.");
1044 			throw new Exception("dub cannot remove locally installed packages.");
1045 		}
1046 
1047 		Package[] packages;
1048 
1049 		// Retrieve packages to be removed.
1050 		foreach(pack; m_packageManager.getPackageIterator(package_id))
1051 			if (m_packageManager.isManagedPackage(pack))
1052 				packages ~= pack;
1053 
1054 		// Check validity of packages to be removed.
1055 		if(packages.empty) {
1056 			throw new Exception("Cannot find package to remove. ("
1057 				~ "id: '" ~ package_id ~ "', location: '" ~ to!string(location) ~ "'"
1058 				~ ")");
1059 		}
1060 
1061 		// Sort package list in ascending version order
1062 		packages.sort!((a, b) => a.version_ < b.version_);
1063 
1064 		immutable idx = resolve_version(packages);
1065 		if (idx == size_t.max)
1066 			return;
1067 		else if (idx != packages.length)
1068 			packages = packages[idx .. idx + 1];
1069 
1070 		logDebug("Removing %s packages.", packages.length);
1071 		foreach(pack; packages) {
1072 			try {
1073 				remove(pack);
1074 				logInfo("Removed %s, version %s.", package_id, pack.version_);
1075 			} catch (Exception e) {
1076 				logError("Failed to remove %s %s: %s", package_id, pack.version_, e.msg);
1077 				logInfo("Continuing with other packages (if any).");
1078 			}
1079 		}
1080 	}
1081 
1082 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1083 	void remove(string package_id, PlacementLocation location, bool force_remove,
1084 				scope size_t delegate(in Package[] packages) resolve_version)
1085 	{
1086 		remove(package_id, location, resolve_version);
1087 	}
1088 
1089 	/** Removes a specific version of a package.
1090 
1091 		Params:
1092 			package_id = Name of the package to be removed
1093 			version_ = Identifying a version or a wild card. If an empty string
1094 				is passed, the package will be removed from the location, if
1095 				there is only one version retrieved. This will throw an
1096 				exception, if there are multiple versions retrieved.
1097 			location_ = Specifies the location to look for the given package
1098 				name/version.
1099 	 */
1100 	void remove(string package_id, string version_, PlacementLocation location)
1101 	{
1102 		remove(package_id, location, (in packages) {
1103 			if (version_ == RemoveVersionWildcard || version_.empty)
1104 				return packages.length;
1105 
1106 			foreach (i, p; packages) {
1107 				if (p.version_ == Version(version_))
1108 					return i;
1109 			}
1110 			throw new Exception("Cannot find package to remove. ("
1111 				~ "id: '" ~ package_id ~ "', version: '" ~ version_ ~ "', location: '" ~ to!string(location) ~ "'"
1112 				~ ")");
1113 		});
1114 	}
1115 
1116 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
1117 	void remove(string package_id, string version_, PlacementLocation location, bool force_remove)
1118 	{
1119 		remove(package_id, version_, location);
1120 	}
1121 
1122 	/** Adds a directory to the list of locally known packages.
1123 
1124 		Forwards to `PackageManager.addLocalPackage`.
1125 
1126 		Params:
1127 			path = Path to the package
1128 			ver = Optional version to associate with the package (can be left
1129 				empty)
1130 			system = Make the package known system wide instead of user wide
1131 				(requires administrator privileges).
1132 
1133 		See_Also: `removeLocalPackage`
1134 	*/
1135 	void addLocalPackage(string path, string ver, bool system)
1136 	{
1137 		if (m_dryRun) return;
1138 		m_packageManager.addLocalPackage(makeAbsolute(path), ver, system ? LocalPackageType.system : LocalPackageType.user);
1139 	}
1140 
1141 	/** Removes a directory from the list of locally known packages.
1142 
1143 		Forwards to `PackageManager.removeLocalPackage`.
1144 
1145 		Params:
1146 			path = Path to the package
1147 			system = Make the package known system wide instead of user wide
1148 				(requires administrator privileges).
1149 
1150 		See_Also: `addLocalPackage`
1151 	*/
1152 	void removeLocalPackage(string path, bool system)
1153 	{
1154 		if (m_dryRun) return;
1155 		m_packageManager.removeLocalPackage(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1156 	}
1157 
1158 	/** Registers a local directory to search for packages to use for satisfying
1159 		dependencies.
1160 
1161 		Params:
1162 			path = Path to a directory containing package directories
1163 			system = Make the package known system wide instead of user wide
1164 				(requires administrator privileges).
1165 
1166 		See_Also: `removeSearchPath`
1167 	*/
1168 	void addSearchPath(string path, bool system)
1169 	{
1170 		if (m_dryRun) return;
1171 		m_packageManager.addSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1172 	}
1173 
1174 	/** Unregisters a local directory search path.
1175 
1176 		Params:
1177 			path = Path to a directory containing package directories
1178 			system = Make the package known system wide instead of user wide
1179 				(requires administrator privileges).
1180 
1181 		See_Also: `addSearchPath`
1182 	*/
1183 	void removeSearchPath(string path, bool system)
1184 	{
1185 		if (m_dryRun) return;
1186 		m_packageManager.removeSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1187 	}
1188 
1189 	/** Queries all package suppliers with the given query string.
1190 
1191 		Returns a list of tuples, where the first entry is the human readable
1192 		name of the package supplier and the second entry is the list of
1193 		matched packages.
1194 
1195 		See_Also: `PackageSupplier.searchPackages`
1196 	*/
1197 	auto searchPackages(string query)
1198 	{
1199 		import std.typecons : Tuple, tuple;
1200 		Tuple!(string, PackageSupplier.SearchResult[])[] results;
1201 		foreach (ps; this.m_packageSuppliers) {
1202 			try
1203 				results ~= tuple(ps.description, ps.searchPackages(query));
1204 			catch (Exception e) {
1205 				logWarn("Searching %s for '%s' failed: %s", ps.description, query, e.msg);
1206 			}
1207 		}
1208 		return results.filter!(tup => tup[1].length);
1209 	}
1210 
1211 	/** Returns a list of all available versions (including branches) for a
1212 		particular package.
1213 
1214 		The list returned is based on the registered package suppliers. Local
1215 		packages are not queried in the search for versions.
1216 
1217 		See_also: `getLatestVersion`
1218 	*/
1219 	Version[] listPackageVersions(string name)
1220 	{
1221 		Version[] versions;
1222 		auto basePackageName = getBasePackageName(name);
1223 		foreach (ps; this.m_packageSuppliers) {
1224 			try versions ~= ps.getVersions(basePackageName);
1225 			catch (Exception e) {
1226 				logWarn("Failed to get versions for package %s on provider %s: %s", name, ps.description, e.msg);
1227 			}
1228 		}
1229 		return versions.sort().uniq.array;
1230 	}
1231 
1232 	/** Returns the latest available version for a particular package.
1233 
1234 		This function returns the latest numbered version of a package. If no
1235 		numbered versions are available, it will return an available branch,
1236 		preferring "~master".
1237 
1238 		Params:
1239 			package_name: The name of the package in question.
1240 			prefer_stable: If set to `true` (the default), returns the latest
1241 				stable version, even if there are newer pre-release versions.
1242 
1243 		See_also: `listPackageVersions`
1244 	*/
1245 	Version getLatestVersion(string package_name, bool prefer_stable = true)
1246 	{
1247 		auto vers = listPackageVersions(package_name);
1248 		enforce(!vers.empty, "Failed to find any valid versions for a package name of '"~package_name~"'.");
1249 		auto final_versions = vers.filter!(v => !v.isBranch && !v.isPreRelease).array;
1250 		if (prefer_stable && final_versions.length) return final_versions[$-1];
1251 		else return vers[$-1];
1252 	}
1253 
1254 	/** Initializes a directory with a package skeleton.
1255 
1256 		Params:
1257 			path = Path of the directory to create the new package in. The
1258 				directory will be created if it doesn't exist.
1259 			deps = List of dependencies to add to the package recipe.
1260 			type = Specifies the type of the application skeleton to use.
1261 			format = Determines the package recipe format to use.
1262 			recipe_callback = Optional callback that can be used to
1263 				customize the recipe before it gets written.
1264 	*/
1265 	void createEmptyPackage(NativePath path, string[] deps, string type,
1266 		PackageFormat format = PackageFormat.sdl,
1267 		scope void delegate(ref PackageRecipe, ref PackageFormat) recipe_callback = null,
1268 		string[] app_args = [])
1269 	{
1270 		if (!path.absolute) path = m_rootPath ~ path;
1271 		path.normalize();
1272 
1273 		string[string] depVers;
1274 		string[] notFound; // keep track of any failed packages in here
1275 		foreach (dep; deps) {
1276 			Version ver;
1277 			try {
1278 				ver = getLatestVersion(dep);
1279 				depVers[dep] = ver.isBranch ? ver.toString() : "~>" ~ ver.toString();
1280 			} catch (Exception e) {
1281 				notFound ~= dep;
1282 			}
1283 		}
1284 
1285 		if(notFound.length > 1){
1286 			throw new Exception(.format("Couldn't find packages: %-(%s, %).", notFound));
1287 		}
1288 		else if(notFound.length == 1){
1289 			throw new Exception(.format("Couldn't find package: %-(%s, %).", notFound));
1290 		}
1291 
1292 		if (m_dryRun) return;
1293 
1294 		initPackage(path, depVers, type, format, recipe_callback);
1295 
1296 		if (!["vibe.d", "deimos", "minimal"].canFind(type)) {
1297 			runCustomInitialization(path, type, app_args);
1298 		}
1299 
1300 		//Act smug to the user.
1301 		logInfo("Successfully created an empty project in '%s'.", path.toNativeString());
1302 	}
1303 
1304 	private void runCustomInitialization(NativePath path, string type, string[] runArgs)
1305 	{
1306 		string packageName = type;
1307 		auto template_pack = m_packageManager.getBestPackage(packageName, ">=0.0.0");
1308 		if (!template_pack) template_pack = m_packageManager.getBestPackage(packageName, "~master");
1309 		if (!template_pack) {
1310 			logInfo("%s is not present, getting and storing it user wide", packageName);
1311 			template_pack = fetch(packageName, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1312 		}
1313 
1314 		Package initSubPackage = m_packageManager.getSubPackage(template_pack, "init-exec", false);
1315 		auto template_dub = new Dub(null, m_packageSuppliers);
1316 		template_dub.loadPackage(initSubPackage);
1317 		auto compiler_binary = this.defaultCompiler;
1318 
1319 		GeneratorSettings settings;
1320 		settings.config = "application";
1321 		settings.compiler = getCompiler(compiler_binary);
1322 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
1323 		settings.buildType = "debug";
1324 		settings.run = true;
1325 		settings.runArgs = runArgs;
1326 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
1327 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
1328 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
1329 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
1330 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
1331 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
1332 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
1333 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
1334 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
1335 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
1336 		initSubPackage.recipe.buildSettings.workingDirectory = path.toNativeString();
1337 		template_dub.generateProject("build", settings);
1338 	}
1339 
1340 	/** Converts the package recipe of the loaded root package to the given format.
1341 
1342 		Params:
1343 			destination_file_ext = The file extension matching the desired
1344 				format. Possible values are "json" or "sdl".
1345 			print_only = Print the converted recipe instead of writing to disk
1346 	*/
1347 	void convertRecipe(string destination_file_ext, bool print_only = false)
1348 	{
1349 		import std.path : extension;
1350 		import std.stdio : stdout;
1351 		import dub.recipe.io : serializePackageRecipe, writePackageRecipe;
1352 
1353 		if (print_only) {
1354 			auto dst = stdout.lockingTextWriter;
1355 			serializePackageRecipe(dst, m_project.rootPackage.rawRecipe, "dub."~destination_file_ext);
1356 			return;
1357 		}
1358 
1359 		auto srcfile = m_project.rootPackage.recipePath;
1360 		auto srcext = srcfile.head.name.extension;
1361 		if (srcext == "."~destination_file_ext) {
1362 			logInfo("Package format is already %s.", destination_file_ext);
1363 			return;
1364 		}
1365 
1366 		writePackageRecipe(srcfile.parentPath ~ ("dub."~destination_file_ext), m_project.rootPackage.rawRecipe);
1367 		removeFile(srcfile);
1368 	}
1369 
1370 	/** Runs DDOX to generate or serve documentation.
1371 
1372 		Params:
1373 			run = If set to true, serves documentation on a local web server.
1374 				Otherwise generates actual HTML files.
1375 			generate_args = Additional command line arguments to pass to
1376 				"ddox generate-html" or "ddox serve-html".
1377 	*/
1378 	void runDdox(bool run, string[] generate_args = null)
1379 	{
1380 		import std.process : browse;
1381 
1382 		if (m_dryRun) return;
1383 
1384 		// allow to choose a custom ddox tool
1385 		auto tool = m_project.rootPackage.recipe.ddoxTool;
1386 		if (tool.empty) tool = "ddox";
1387 
1388 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
1389 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
1390 		if (!tool_pack) {
1391 			logInfo("%s is not present, getting and storing it user wide", tool);
1392 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1393 		}
1394 
1395 		auto ddox_dub = new Dub(null, m_packageSuppliers);
1396 		ddox_dub.loadPackage(tool_pack.path);
1397 		ddox_dub.upgrade(UpgradeOptions.select);
1398 
1399 		auto compiler_binary = this.defaultCompiler;
1400 
1401 		GeneratorSettings settings;
1402 		settings.config = "application";
1403 		settings.compiler = getCompiler(compiler_binary); // TODO: not using --compiler ???
1404 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary, m_defaultArchitecture);
1405 		settings.buildType = "debug";
1406 		if (m_defaultLowMemory) settings.buildSettings.options |= BuildOption.lowmem;
1407 		if (m_defaultEnvironments) settings.buildSettings.addEnvironments(m_defaultEnvironments);
1408 		if (m_defaultBuildEnvironments) settings.buildSettings.addBuildEnvironments(m_defaultBuildEnvironments);
1409 		if (m_defaultRunEnvironments) settings.buildSettings.addRunEnvironments(m_defaultRunEnvironments);
1410 		if (m_defaultPreGenerateEnvironments) settings.buildSettings.addPreGenerateEnvironments(m_defaultPreGenerateEnvironments);
1411 		if (m_defaultPostGenerateEnvironments) settings.buildSettings.addPostGenerateEnvironments(m_defaultPostGenerateEnvironments);
1412 		if (m_defaultPreBuildEnvironments) settings.buildSettings.addPreBuildEnvironments(m_defaultPreBuildEnvironments);
1413 		if (m_defaultPostBuildEnvironments) settings.buildSettings.addPostBuildEnvironments(m_defaultPostBuildEnvironments);
1414 		if (m_defaultPreRunEnvironments) settings.buildSettings.addPreRunEnvironments(m_defaultPreRunEnvironments);
1415 		if (m_defaultPostRunEnvironments) settings.buildSettings.addPostRunEnvironments(m_defaultPostRunEnvironments);
1416 		settings.run = true;
1417 
1418 		auto filterargs = m_project.rootPackage.recipe.ddoxFilterArgs.dup;
1419 		if (filterargs.empty) filterargs = ["--min-protection=Protected", "--only-documented"];
1420 
1421 		settings.runArgs = "filter" ~ filterargs ~ "docs.json";
1422 		ddox_dub.generateProject("build", settings);
1423 
1424 		auto p = tool_pack.path;
1425 		p.endsWithSlash = true;
1426 		auto tool_path = p.toNativeString();
1427 
1428 		if (run) {
1429 			settings.runArgs = ["serve-html", "--navigation-type=ModuleTree", "docs.json", "--web-file-dir="~tool_path~"public"] ~ generate_args;
1430 			browse("http://127.0.0.1:8080/");
1431 		} else {
1432 			settings.runArgs = ["generate-html", "--navigation-type=ModuleTree", "docs.json", "docs"] ~ generate_args;
1433 		}
1434 		ddox_dub.generateProject("build", settings);
1435 
1436 		if (!run) {
1437 			// TODO: ddox should copy those files itself
1438 			version(Windows) runCommand(`xcopy /S /D "`~tool_path~`public\*" docs\`);
1439 			else runCommand("rsync -ru '"~tool_path~"public/' docs/");
1440 		}
1441 	}
1442 
1443 	private void updatePackageSearchPath()
1444 	{
1445 		// TODO: Remove once `overrideSearchPath` is removed
1446 		if (!m_overrideSearchPath.empty) {
1447 			m_packageManager._disableDefaultSearchPaths = true;
1448 			m_packageManager.searchPath = [m_overrideSearchPath];
1449 			return;
1450 		}
1451 
1452 		auto p = environment.get("DUBPATH");
1453 		NativePath[] paths;
1454 
1455 		version(Windows) enum pathsep = ";";
1456 		else enum pathsep = ":";
1457 		if (p.length) paths ~= p.split(pathsep).map!(p => NativePath(p))().array();
1458 		m_packageManager._disableDefaultSearchPaths = false;
1459 		m_packageManager.searchPath = paths;
1460 	}
1461 
1462 	private void determineDefaultCompiler()
1463 	{
1464 		import std.file : thisExePath;
1465 		import std.path : buildPath, dirName, expandTilde, isAbsolute, isDirSeparator;
1466 		import std.range : front;
1467 
1468 		// Env takes precedence
1469 		if (auto envCompiler = environment.get("DC"))
1470 			m_defaultCompiler = envCompiler;
1471 		else
1472 			m_defaultCompiler = m_config.defaultCompiler.expandTilde;
1473 		if (m_defaultCompiler.length && m_defaultCompiler.isAbsolute)
1474 			return;
1475 
1476 		static immutable BinaryPrefix = `$DUB_BINARY_PATH`;
1477 		if(m_defaultCompiler.startsWith(BinaryPrefix))
1478 		{
1479 			m_defaultCompiler = thisExePath().dirName() ~ m_defaultCompiler[BinaryPrefix.length .. $];
1480 			return;
1481 		}
1482 
1483 		if (!find!isDirSeparator(m_defaultCompiler).empty)
1484 			throw new Exception("defaultCompiler specified in a DUB config file cannot use an unqualified relative path:\n\n" ~ m_defaultCompiler ~
1485 			"\n\nUse \"$DUB_BINARY_PATH/../path/you/want\" instead.");
1486 
1487 		version (Windows) enum sep = ";", exe = ".exe";
1488 		version (Posix) enum sep = ":", exe = "";
1489 
1490 		auto compilers = ["dmd", "gdc", "gdmd", "ldc2", "ldmd2"];
1491 		// If a compiler name is specified, look for it next to dub.
1492 		// Otherwise, look for any of the common compilers adjacent to dub.
1493 		if (m_defaultCompiler.length)
1494 		{
1495 			string compilerPath = buildPath(thisExePath().dirName(), m_defaultCompiler ~ exe);
1496 			if (existsFile(compilerPath))
1497 			{
1498 				m_defaultCompiler = compilerPath;
1499 				return;
1500 			}
1501 		}
1502 		else
1503 		{
1504 			auto nextFound = compilers.find!(bin => existsFile(buildPath(thisExePath().dirName(), bin ~ exe)));
1505 			if (!nextFound.empty)
1506 			{
1507 				m_defaultCompiler = buildPath(thisExePath().dirName(),  nextFound.front ~ exe);
1508 				return;
1509 			}
1510 		}
1511 
1512 		// If nothing found next to dub, search the user's PATH, starting
1513 		// with the compiler name from their DUB config file, if specified.
1514 		auto paths = environment.get("PATH", "").splitter(sep).map!NativePath;
1515 		if (m_defaultCompiler.length && paths.canFind!(p => existsFile(p ~ (m_defaultCompiler~exe))))
1516 			return;
1517 		foreach (p; paths) {
1518 			auto res = compilers.find!(bin => existsFile(p ~ (bin~exe)));
1519 			if (!res.empty) {
1520 				m_defaultCompiler = res.front;
1521 				return;
1522 			}
1523 		}
1524 		m_defaultCompiler = compilers[0];
1525 	}
1526 
1527 	unittest
1528 	{
1529 		import std.path: buildPath, absolutePath;
1530 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
1531 		immutable olddc = environment.get("DC", null);
1532 		immutable oldpath = environment.get("PATH", null);
1533 		immutable testdir = "test-determineDefaultCompiler";
1534 		void repairenv(string name, string var)
1535 		{
1536 			if (var !is null)
1537 				environment[name] = var;
1538 			else if (name in environment)
1539 				environment.remove(name);
1540 		}
1541 		scope (exit) repairenv("DC", olddc);
1542 		scope (exit) repairenv("PATH", oldpath);
1543 		scope (exit) rmdirRecurse(testdir);
1544 
1545 		version (Windows) enum sep = ";", exe = ".exe";
1546 		version (Posix) enum sep = ":", exe = "";
1547 
1548 		immutable dmdpath = testdir.buildPath("dmd", "bin");
1549 		immutable ldcpath = testdir.buildPath("ldc", "bin");
1550 		mkdirRecurse(dmdpath);
1551 		mkdirRecurse(ldcpath);
1552 		immutable dmdbin = dmdpath.buildPath("dmd"~exe);
1553 		immutable ldcbin = ldcpath.buildPath("ldc2"~exe);
1554 		std.file.write(dmdbin, null);
1555 		std.file.write(ldcbin, null);
1556 
1557 		environment["DC"] = dmdbin.absolutePath();
1558 		dub.determineDefaultCompiler();
1559 		assert(dub.m_defaultCompiler == dmdbin.absolutePath());
1560 
1561 		environment["DC"] = "dmd";
1562 		environment["PATH"] = dmdpath ~ sep ~ ldcpath;
1563 		dub.determineDefaultCompiler();
1564 		assert(dub.m_defaultCompiler == "dmd");
1565 
1566 		environment["DC"] = "ldc2";
1567 		environment["PATH"] = dmdpath ~ sep ~ ldcpath;
1568 		dub.determineDefaultCompiler();
1569 		assert(dub.m_defaultCompiler == "ldc2");
1570 
1571 		environment.remove("DC");
1572 		environment["PATH"] = ldcpath ~ sep ~ dmdpath;
1573 		dub.determineDefaultCompiler();
1574 		assert(dub.m_defaultCompiler == "ldc2");
1575 	}
1576 
1577 	private NativePath makeAbsolute(NativePath p) const { return p.absolute ? p : m_rootPath ~ p; }
1578 	private NativePath makeAbsolute(string p) const { return makeAbsolute(NativePath(p)); }
1579 }
1580 
1581 
1582 /// Option flags for `Dub.fetch`
1583 enum FetchOptions
1584 {
1585 	none = 0,
1586 	forceBranchUpgrade = 1<<0,
1587 	usePrerelease = 1<<1,
1588 	forceRemove = 1<<2, /// Deprecated, does nothing.
1589 	printOnly = 1<<3,
1590 }
1591 
1592 /// Option flags for `Dub.upgrade`
1593 enum UpgradeOptions
1594 {
1595 	none = 0,
1596 	upgrade = 1<<1, /// Upgrade existing packages
1597 	preRelease = 1<<2, /// inclde pre-release versions in upgrade
1598 	forceRemove = 1<<3, /// Deprecated, does nothing.
1599 	select = 1<<4, /// Update the dub.selections.json file with the upgraded versions
1600 	dryRun = 1<<5, /// Instead of downloading new packages, just print a message to notify the user of their existence
1601 	/*deprecated*/ printUpgradesOnly = dryRun, /// deprecated, use dryRun instead
1602 	/*deprecated*/ useCachedResult = 1<<6, /// deprecated, has no effect
1603 	noSaveSelections = 1<<7, /// Don't store updated selections on disk
1604 }
1605 
1606 /// Determines which of the default package suppliers are queried for packages.
1607 enum SkipPackageSuppliers {
1608 	none,       /// Uses all configured package suppliers.
1609 	standard,   /// Does not use the default package suppliers (`defaultPackageSuppliers`).
1610 	configured, /// Does not use default suppliers or suppliers configured in DUB's configuration file
1611 	all         /// Uses only manually specified package suppliers.
1612 }
1613 
1614 private class DependencyVersionResolver : DependencyResolver!(Dependency, Dependency) {
1615 	protected {
1616 		Dub m_dub;
1617 		UpgradeOptions m_options;
1618 		Dependency[][string] m_packageVersions;
1619 		Package[string] m_remotePackages;
1620 		SelectedVersions m_selectedVersions;
1621 		Package m_rootPackage;
1622 		bool[string] m_packagesToUpgrade;
1623 		Package[PackageDependency] m_packages;
1624 		TreeNodes[][TreeNode] m_children;
1625 	}
1626 
1627 
1628 	this(Dub dub, UpgradeOptions options)
1629 	{
1630 		m_dub = dub;
1631 		m_options = options;
1632 	}
1633 
1634 	void addPackageToUpgrade(string name)
1635 	{
1636 		m_packagesToUpgrade[name] = true;
1637 	}
1638 
1639 	Dependency[string] resolve(Package root, SelectedVersions selected_versions)
1640 	{
1641 		m_rootPackage = root;
1642 		m_selectedVersions = selected_versions;
1643 		return super.resolve(TreeNode(root.name, Dependency(root.version_)), (m_options & UpgradeOptions.printUpgradesOnly) == 0);
1644 	}
1645 
1646 	protected bool isFixedPackage(string pack)
1647 	{
1648 		return m_packagesToUpgrade !is null && pack !in m_packagesToUpgrade;
1649 	}
1650 
1651 	protected override Dependency[] getAllConfigs(string pack)
1652 	{
1653 		if (auto pvers = pack in m_packageVersions)
1654 			return *pvers;
1655 
1656 		if ((!(m_options & UpgradeOptions.upgrade) || isFixedPackage(pack)) && m_selectedVersions.hasSelectedVersion(pack)) {
1657 			auto ret = [m_selectedVersions.getSelectedVersion(pack)];
1658 			logDiagnostic("Using fixed selection %s %s", pack, ret[0]);
1659 			m_packageVersions[pack] = ret;
1660 			return ret;
1661 		}
1662 
1663 		logDiagnostic("Search for versions of %s (%s package suppliers)", pack, m_dub.m_packageSuppliers.length);
1664 		Version[] versions;
1665 		foreach (p; m_dub.packageManager.getPackageIterator(pack))
1666 			versions ~= p.version_;
1667 
1668 		foreach (ps; m_dub.m_packageSuppliers) {
1669 			try {
1670 				auto vers = ps.getVersions(pack);
1671 				vers.reverse();
1672 				if (!vers.length) {
1673 					logDiagnostic("No versions for %s for %s", pack, ps.description);
1674 					continue;
1675 				}
1676 
1677 				versions ~= vers;
1678 				break;
1679 			} catch (Exception e) {
1680 				logWarn("Package %s not found in %s: %s", pack, ps.description, e.msg);
1681 				logDebug("Full error: %s", e.toString().sanitize);
1682 			}
1683 		}
1684 
1685 		// sort by version, descending, and remove duplicates
1686 		versions = versions.sort!"a>b".uniq.array;
1687 
1688 		// move pre-release versions to the back of the list if no preRelease flag is given
1689 		if (!(m_options & UpgradeOptions.preRelease))
1690 			versions = versions.filter!(v => !v.isPreRelease).array ~ versions.filter!(v => v.isPreRelease).array;
1691 
1692 		// filter out invalid/unreachable dependency specs
1693 		versions = versions.filter!((v) {
1694 				bool valid = getPackage(pack, Dependency(v)) !is null;
1695 				if (!valid) logDiagnostic("Excluding invalid dependency specification %s %s from dependency resolution process.", pack, v);
1696 				return valid;
1697 			}).array;
1698 
1699 		if (!versions.length) logDiagnostic("Nothing found for %s", pack);
1700 		else logDiagnostic("Return for %s: %s", pack, versions);
1701 
1702 		auto ret = versions.map!(v => Dependency(v)).array;
1703 		m_packageVersions[pack] = ret;
1704 		return ret;
1705 	}
1706 
1707 	protected override Dependency[] getSpecificConfigs(string pack, TreeNodes nodes)
1708 	{
1709 		if (!nodes.configs.path.empty || !nodes.configs.repository.empty) {
1710 			if (getPackage(pack, nodes.configs)) return [nodes.configs];
1711 			else return null;
1712 		}
1713 		else return null;
1714 	}
1715 
1716 
1717 	protected override TreeNodes[] getChildren(TreeNode node)
1718 	{
1719 		if (auto pc = node in m_children)
1720 			return *pc;
1721 		auto ret = getChildrenRaw(node);
1722 		m_children[node] = ret;
1723 		return ret;
1724 	}
1725 
1726 	private final TreeNodes[] getChildrenRaw(TreeNode node)
1727 	{
1728 		import std.array : appender;
1729 		auto ret = appender!(TreeNodes[]);
1730 		auto pack = getPackage(node.pack, node.config);
1731 		if (!pack) {
1732 			// this can hapen when the package description contains syntax errors
1733 			logDebug("Invalid package in dependency tree: %s %s", node.pack, node.config);
1734 			return null;
1735 		}
1736 		auto basepack = pack.basePackage;
1737 
1738 		foreach (d; pack.getAllDependenciesRange()) {
1739 			auto dbasename = getBasePackageName(d.name);
1740 
1741 			// detect dependencies to the root package (or sub packages thereof)
1742 			if (dbasename == basepack.name) {
1743 				auto absdeppath = d.spec.mapToPath(pack.path).path;
1744 				absdeppath.endsWithSlash = true;
1745 				auto subpack = m_dub.m_packageManager.getSubPackage(basepack, getSubPackageName(d.name), true);
1746 				if (subpack) {
1747 					auto desireddeppath = basepack.path;
1748 					desireddeppath.endsWithSlash = true;
1749 
1750 					auto altdeppath = d.name == dbasename ? basepack.path : subpack.path;
1751 					altdeppath.endsWithSlash = true;
1752 
1753 					if (!d.spec.path.empty && absdeppath != desireddeppath)
1754 						logWarn("Warning: Sub package %s, referenced by %s %s must be referenced using the path to its base package",
1755 							subpack.name, pack.name, pack.version_);
1756 
1757 					enforce(d.spec.path.empty || absdeppath == desireddeppath || absdeppath == altdeppath,
1758 						format("Dependency from %s to %s uses wrong path: %s vs. %s",
1759 							node.pack, subpack.name, absdeppath.toNativeString(), desireddeppath.toNativeString()));
1760 				}
1761 				ret ~= TreeNodes(d.name, node.config);
1762 				continue;
1763 			}
1764 
1765 			DependencyType dt;
1766 			if (d.spec.optional) {
1767 				if (d.spec.default_) dt = DependencyType.optionalDefault;
1768 				else dt = DependencyType.optional;
1769 			} else dt = DependencyType.required;
1770 
1771 			Dependency dspec = d.spec.mapToPath(pack.path);
1772 
1773 			// if not upgrading, use the selected version
1774 			if (!(m_options & UpgradeOptions.upgrade) && m_selectedVersions && m_selectedVersions.hasSelectedVersion(dbasename))
1775 				dspec = m_selectedVersions.getSelectedVersion(dbasename);
1776 
1777 			// keep selected optional dependencies and avoid non-selected optional-default dependencies by default
1778 			if (m_selectedVersions && !m_selectedVersions.bare) {
1779 				if (dt == DependencyType.optionalDefault && !m_selectedVersions.hasSelectedVersion(dbasename))
1780 					dt = DependencyType.optional;
1781 				else if (dt == DependencyType.optional && m_selectedVersions.hasSelectedVersion(dbasename))
1782 					dt = DependencyType.optionalDefault;
1783 			}
1784 
1785 			ret ~= TreeNodes(d.name, dspec, dt);
1786 		}
1787 		return ret.data;
1788 	}
1789 
1790 	protected override bool matches(Dependency configs, Dependency config)
1791 	{
1792 		if (!configs.path.empty) return configs.path == config.path;
1793 		return configs.merge(config).valid;
1794 	}
1795 
1796 	private Package getPackage(string name, Dependency dep)
1797 	{
1798 		auto key = PackageDependency(name, dep);
1799 		if (auto pp = key in m_packages)
1800 			return *pp;
1801 		auto p = getPackageRaw(name, dep);
1802 		m_packages[key] = p;
1803 		return p;
1804 	}
1805 
1806 	private Package getPackageRaw(string name, Dependency dep)
1807 	{
1808 		auto basename = getBasePackageName(name);
1809 
1810 		// for sub packages, first try to get them from the base package
1811 		if (basename != name) {
1812 			auto subname = getSubPackageName(name);
1813 			auto basepack = getPackage(basename, dep);
1814 			if (!basepack) return null;
1815 			if (auto sp = m_dub.m_packageManager.getSubPackage(basepack, subname, true)) {
1816 				return sp;
1817 			} else if (!basepack.subPackages.canFind!(p => p.path.length)) {
1818 				// note: external sub packages are handled further below
1819 				auto spr = basepack.getInternalSubPackage(subname);
1820 				if (!spr.isNull) {
1821 					auto sp = new Package(spr.get, basepack.path, basepack);
1822 					m_remotePackages[sp.name] = sp;
1823 					return sp;
1824 				} else {
1825 					logDiagnostic("Sub package %s doesn't exist in %s %s.", name, basename, dep.version_);
1826 					return null;
1827 				}
1828 			} else if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) {
1829 				return ret;
1830 			} else {
1831 				logDiagnostic("External sub package %s %s not found.", name, dep.version_);
1832 				return null;
1833 			}
1834 		}
1835 
1836 		// shortcut if the referenced package is the root package
1837 		if (basename == m_rootPackage.basePackage.name)
1838 			return m_rootPackage.basePackage;
1839 
1840 		if (!dep.repository.empty) {
1841 			auto ret = m_dub.packageManager.loadSCMPackage(name, dep);
1842 			return ret !is null && dep.matches(ret.version_) ? ret : null;
1843 		} else if (!dep.path.empty) {
1844 			try {
1845 				auto ret = m_dub.packageManager.getOrLoadPackage(dep.path);
1846 				if (dep.matches(ret.version_)) return ret;
1847 			} catch (Exception e) {
1848 				logDiagnostic("Failed to load path based dependency %s: %s", name, e.msg);
1849 				logDebug("Full error: %s", e.toString().sanitize);
1850 				return null;
1851 			}
1852 		}
1853 
1854 		if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep))
1855 			return ret;
1856 
1857 		auto key = name ~ ":" ~ dep.version_.toString();
1858 		if (auto ret = key in m_remotePackages)
1859 			return *ret;
1860 
1861 		auto prerelease = (m_options & UpgradeOptions.preRelease) != 0;
1862 
1863 		auto rootpack = name.split(":")[0];
1864 
1865 		foreach (ps; m_dub.m_packageSuppliers) {
1866 			if (rootpack == name) {
1867 				try {
1868 					auto desc = ps.fetchPackageRecipe(name, dep, prerelease);
1869 					if (desc.type == Json.Type.null_)
1870 						continue;
1871 					auto ret = new Package(desc);
1872 					m_remotePackages[key] = ret;
1873 					return ret;
1874 				} catch (Exception e) {
1875 					logDiagnostic("Metadata for %s %s could not be downloaded from %s: %s", name, dep, ps.description, e.msg);
1876 					logDebug("Full error: %s", e.toString().sanitize);
1877 				}
1878 			} else {
1879 				logDiagnostic("Package %s not found in base package description (%s). Downloading whole package.", name, dep.version_.toString());
1880 				try {
1881 					FetchOptions fetchOpts;
1882 					fetchOpts |= prerelease ? FetchOptions.usePrerelease : FetchOptions.none;
1883 					m_dub.fetch(rootpack, dep, m_dub.defaultPlacementLocation, fetchOpts, "need sub package description");
1884 					auto ret = m_dub.m_packageManager.getBestPackage(name, dep);
1885 					if (!ret) {
1886 						logWarn("Package %s %s doesn't have a sub package %s", rootpack, dep.version_, name);
1887 						return null;
1888 					}
1889 					m_remotePackages[key] = ret;
1890 					return ret;
1891 				} catch (Exception e) {
1892 					logDiagnostic("Package %s could not be downloaded from %s: %s", rootpack, ps.description, e.msg);
1893 					logDebug("Full error: %s", e.toString().sanitize);
1894 				}
1895 			}
1896 		}
1897 
1898 		m_remotePackages[key] = null;
1899 
1900 		logWarn("Package %s %s could not be loaded either locally, or from the configured package registries.", name, dep);
1901 		return null;
1902 	}
1903 }
1904 
1905 private struct SpecialDirs {
1906 	NativePath temp;
1907 	NativePath userSettings;
1908 	NativePath systemSettings;
1909 	NativePath localRepository;
1910 }
1911 
1912 private class DubConfig {
1913 	private {
1914 		DubConfig m_parentConfig;
1915 		Json m_data;
1916 	}
1917 
1918 	this(Json data, DubConfig parent_config)
1919 	{
1920 		m_data = data;
1921 		m_parentConfig = parent_config;
1922 	}
1923 
1924 	@property string[] registryURLs()
1925 	{
1926 		string[] ret;
1927 		if (auto pv = "registryUrls" in m_data)
1928 			ret = (*pv).deserializeJson!(string[]);
1929 		if (m_parentConfig) ret ~= m_parentConfig.registryURLs;
1930 		return ret;
1931 	}
1932 
1933 	@property SkipPackageSuppliers skipRegistry()
1934 	{
1935 		if(auto pv = "skipRegistry" in m_data)
1936 			return to!SkipPackageSuppliers((*pv).get!string);
1937 
1938 		if (m_parentConfig)
1939 			return m_parentConfig.skipRegistry;
1940 
1941 		return SkipPackageSuppliers.none;
1942 	}
1943 
1944 	@property NativePath[] customCachePaths()
1945 	{
1946 		import std.algorithm.iteration : map;
1947 		import std.array : array;
1948 
1949 		NativePath[] ret;
1950 		if (auto pv = "customCachePaths" in m_data)
1951 			ret = (*pv).deserializeJson!(string[])
1952 				.map!(s => NativePath(s))
1953 				.array;
1954 		if (m_parentConfig)
1955 			ret ~= m_parentConfig.customCachePaths;
1956 		return ret;
1957 	}
1958 
1959 	@property string defaultCompiler()
1960 	const {
1961 		if (auto pv = "defaultCompiler" in m_data)
1962 			return pv.get!string;
1963 		if (m_parentConfig) return m_parentConfig.defaultCompiler;
1964 		return null;
1965 	}
1966 
1967 	@property string defaultArchitecture()
1968 	const {
1969 		if(auto pv = "defaultArchitecture" in m_data)
1970 			return (*pv).get!string;
1971 		if (m_parentConfig) return m_parentConfig.defaultArchitecture;
1972 		return null;
1973 	}
1974 
1975 	@property bool defaultLowMemory()
1976 	const {
1977 		if(auto pv = "defaultLowMemory" in m_data)
1978 			return (*pv).get!bool;
1979 		if (m_parentConfig) return m_parentConfig.defaultLowMemory;
1980 		return false;
1981 	}
1982 
1983 	@property string[string] defaultEnvironments()
1984 	const {
1985 		if (auto pv = "defaultEnvironments" in m_data)
1986 			return deserializeJson!(string[string])(*cast(Json*)pv);
1987 		if (m_parentConfig) return m_parentConfig.defaultEnvironments;
1988 		return null;
1989 	}
1990 
1991 	@property string[string] defaultBuildEnvironments()
1992 	const {
1993 		if (auto pv = "defaultBuildEnvironments" in m_data)
1994 			return deserializeJson!(string[string])(*cast(Json*)pv);
1995 		if (m_parentConfig) return m_parentConfig.defaultBuildEnvironments;
1996 		return null;
1997 	}
1998 
1999 	@property string[string] defaultRunEnvironments()
2000 	const {
2001 		if (auto pv = "defaultRunEnvironments" in m_data)
2002 			return deserializeJson!(string[string])(*cast(Json*)pv);
2003 		if (m_parentConfig) return m_parentConfig.defaultRunEnvironments;
2004 		return null;
2005 	}
2006 
2007 	@property string[string] defaultPreGenerateEnvironments()
2008 	const {
2009 		if (auto pv = "defaultPreGenerateEnvironments" in m_data)
2010 			return deserializeJson!(string[string])(*cast(Json*)pv);
2011 		if (m_parentConfig) return m_parentConfig.defaultPreGenerateEnvironments;
2012 		return null;
2013 	}
2014 
2015 	@property string[string] defaultPostGenerateEnvironments()
2016 	const {
2017 		if (auto pv = "defaultPostGenerateEnvironments" in m_data)
2018 			return deserializeJson!(string[string])(*cast(Json*)pv);
2019 		if (m_parentConfig) return m_parentConfig.defaultPostGenerateEnvironments;
2020 		return null;
2021 	}
2022 
2023 	@property string[string] defaultPreBuildEnvironments()
2024 	const {
2025 		if (auto pv = "defaultPreBuildEnvironments" in m_data)
2026 			return deserializeJson!(string[string])(*cast(Json*)pv);
2027 		if (m_parentConfig) return m_parentConfig.defaultPreBuildEnvironments;
2028 		return null;
2029 	}
2030 
2031 	@property string[string] defaultPostBuildEnvironments()
2032 	const {
2033 		if (auto pv = "defaultPostBuildEnvironments" in m_data)
2034 			return deserializeJson!(string[string])(*cast(Json*)pv);
2035 		if (m_parentConfig) return m_parentConfig.defaultPostBuildEnvironments;
2036 		return null;
2037 	}
2038 
2039 	@property string[string] defaultPreRunEnvironments()
2040 	const {
2041 		if (auto pv = "defaultPreRunEnvironments" in m_data)
2042 			return deserializeJson!(string[string])(*cast(Json*)pv);
2043 		if (m_parentConfig) return m_parentConfig.defaultPreRunEnvironments;
2044 		return null;
2045 	}
2046 
2047 	@property string[string] defaultPostRunEnvironments()
2048 	const {
2049 		if (auto pv = "defaultPostRunEnvironments" in m_data)
2050 			return deserializeJson!(string[string])(*cast(Json*)pv);
2051 		if (m_parentConfig) return m_parentConfig.defaultPostRunEnvironments;
2052 		return null;
2053 	}
2054 }