1 /**
2 	A package manager.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff, 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff, Sönke Ludwig
7 */
8 module dub.dub;
9 
10 import dub.compilers.compiler;
11 import dub.dependency;
12 import dub.dependencyresolver;
13 import dub.internal.utils;
14 import dub.internal.vibecompat.core.file;
15 import dub.internal.vibecompat.core.log;
16 import dub.internal.vibecompat.data.json;
17 import dub.internal.vibecompat.inet.url;
18 import dub.package_;
19 import dub.packagemanager;
20 import dub.packagesupplier;
21 import dub.project;
22 import dub.generators.generator;
23 import dub.init;
24 
25 import std.algorithm;
26 import std.array : array, replace;
27 import std.conv : to;
28 import std.exception : enforce;
29 import std.file;
30 import std.process : environment;
31 import std.range : empty;
32 import std..string;
33 import std.encoding : sanitize;
34 
35 // Workaround for libcurl liker errors when building with LDC
36 version (LDC) pragma(lib, "curl");
37 
38 // Set output path and options for coverage reports
39 version (DigitalMars) version (D_Coverage) static if (__VERSION__ >= 2068)
40 {
41 	shared static this()
42 	{
43 		import core.runtime, std.file, std.path, std.stdio;
44 		dmd_coverSetMerge(true);
45 		auto path = buildPath(dirName(thisExePath()), "../cov");
46 		if (!path.exists)
47 			mkdir(path);
48 		dmd_coverDestPath(path);
49 	}
50 }
51 
52 static this()
53 {
54 	import dub.compilers.dmd : DMDCompiler;
55 	import dub.compilers.gdc : GDCCompiler;
56 	import dub.compilers.ldc : LDCCompiler;
57 	registerCompiler(new DMDCompiler);
58 	registerCompiler(new GDCCompiler);
59 	registerCompiler(new LDCCompiler);
60 }
61 
62 /// The URL to the official package registry.
63 enum defaultRegistryURL = "https://code.dlang.org/";
64 enum fallbackRegistryURLs = [
65 	// fallback in case of HTTPS problems
66 	"http://code.dlang.org/",
67 	"https://code-mirror.dlang.io/",
68 	"https://code-mirror2.dlang.io/",
69 	"https://dub-registry.herokuapp.com/",
70 ];
71 
72 /** Returns a default list of package suppliers.
73 
74 	This will contain a single package supplier that points to the official
75 	package registry.
76 
77 	See_Also: `defaultRegistryURL`
78 */
79 PackageSupplier[] defaultPackageSuppliers()
80 {
81 	logDiagnostic("Using dub registry url '%s'", defaultRegistryURL);
82 	return [
83 		new FallbackPackageSupplier(
84 			new RegistryPackageSupplier(URL(defaultRegistryURL)),
85 			fallbackRegistryURLs.map!(x => cast(PackageSupplier) new RegistryPackageSupplier(URL(x))).array
86 		)
87 	];
88 }
89 
90 
91 /** Provides a high-level entry point for DUB's functionality.
92 
93 	This class provides means to load a certain project (a root package with
94 	all of its dependencies) and to perform high-level operations as found in
95 	the command line interface.
96 */
97 class Dub {
98 	private {
99 		bool m_dryRun = false;
100 		PackageManager m_packageManager;
101 		PackageSupplier[] m_packageSuppliers;
102 		NativePath m_rootPath;
103 		SpecialDirs m_dirs;
104 		DubConfig m_config;
105 		NativePath m_projectPath;
106 		Project m_project;
107 		NativePath m_overrideSearchPath;
108 		string m_defaultCompiler;
109 		string m_defaultArchitecture;
110 	}
111 
112 	/** The default placement location of fetched packages.
113 
114 		This property can be altered, so that packages which are downloaded as part
115 		of the normal upgrade process are stored in a certain location. This is
116 		how the "--local" and "--system" command line switches operate.
117 	*/
118 	PlacementLocation defaultPlacementLocation = PlacementLocation.user;
119 
120 
121 	/** Initializes the instance for use with a specific root package.
122 
123 		Note that a package still has to be loaded using one of the
124 		`loadPackage` overloads.
125 
126 		Params:
127 			root_path = Path to the root package
128 			additional_package_suppliers = A list of package suppliers to try
129 				before the suppliers found in the configurations files and the
130 				`defaultPackageSuppliers`.
131 			skip_registry = Can be used to skip using the configured package
132 				suppliers, as well as the default suppliers.
133 	*/
134 	this(string root_path = ".", PackageSupplier[] additional_package_suppliers = null,
135 			SkipPackageSuppliers skip_registry = SkipPackageSuppliers.none)
136 	{
137 		m_rootPath = NativePath(root_path);
138 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
139 
140 		init();
141 
142 		PackageSupplier[] ps = additional_package_suppliers;
143 
144 		if (skip_registry < SkipPackageSuppliers.all)
145 		{
146 			ps ~= environment.get("DUB_REGISTRY", null)
147 				.splitter(";")
148 				.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url)))
149 				.array;
150 		}
151 
152 		if (skip_registry < SkipPackageSuppliers.configured)
153 		{
154 			ps ~= m_config.registryURLs
155 				.map!(url => cast(PackageSupplier)new RegistryPackageSupplier(URL(url)))
156 				.array;
157 		}
158 
159 		if (skip_registry < SkipPackageSuppliers.standard)
160 			ps ~= defaultPackageSuppliers();
161 
162 		m_packageSuppliers = ps;
163 		m_packageManager = new PackageManager(m_dirs.localRepository, m_dirs.systemSettings);
164 		updatePackageSearchPath();
165 	}
166 
167 	unittest
168 	{
169 		scope (exit) environment.remove("DUB_REGISTRY");
170 		auto dub = new Dub(".", null, SkipPackageSuppliers.configured);
171 		assert(dub.m_packageSuppliers.length == 0);
172 		environment["DUB_REGISTRY"] = "http://example.com/";
173 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
174 		logInfo("%s", dub.m_packageSuppliers);
175 		assert(dub.m_packageSuppliers.length == 1);
176 		environment["DUB_REGISTRY"] = "http://example.com/;http://foo.com/";
177 		dub = new Dub(".", null, SkipPackageSuppliers.configured);
178 		assert(dub.m_packageSuppliers.length == 2);
179 		dub = new Dub(".", [new RegistryPackageSupplier(URL("http://bar.com/"))], SkipPackageSuppliers.configured);
180 		assert(dub.m_packageSuppliers.length == 3);
181 	}
182 
183 	/** Initializes the instance with a single package search path, without
184 		loading a package.
185 
186 		This constructor corresponds to the "--bare" option of the command line
187 		interface. Use
188 	*/
189 	this(NativePath override_path)
190 	{
191 		init();
192 		m_overrideSearchPath = override_path;
193 		m_packageManager = new PackageManager(NativePath(), NativePath(), false);
194 		updatePackageSearchPath();
195 	}
196 
197 	private void init()
198 	{
199 		import std.file : tempDir;
200 		version(Windows) {
201 			m_dirs.systemSettings = NativePath(environment.get("ProgramData")) ~ "dub/";
202 			immutable appDataDir = environment.get("APPDATA");
203 			m_dirs.userSettings = NativePath(appDataDir) ~ "dub/";
204 			m_dirs.localRepository = NativePath(environment.get("LOCALAPPDATA", appDataDir)) ~ "dub";
205 
206 			migrateRepositoryFromRoaming(m_dirs.userSettings ~"\\packages", m_dirs.localRepository ~ "\\packages");
207 
208 		} else version(Posix){
209 			m_dirs.systemSettings = NativePath("/var/lib/dub/");
210 			m_dirs.userSettings = NativePath(environment.get("HOME")) ~ ".dub/";
211 			if (!m_dirs.userSettings.absolute)
212 				m_dirs.userSettings = NativePath(getcwd()) ~ m_dirs.userSettings;
213 			m_dirs.localRepository = m_dirs.userSettings;
214 		}
215 
216 		m_dirs.temp = NativePath(tempDir);
217 
218 		m_config = new DubConfig(jsonFromFile(m_dirs.systemSettings ~ "settings.json", true), m_config);
219 		m_config = new DubConfig(jsonFromFile(NativePath(thisExePath).parentPath ~ "../etc/dub/settings.json", true), m_config);
220 		m_config = new DubConfig(jsonFromFile(m_dirs.userSettings ~ "settings.json", true), m_config);
221 
222 		determineDefaultCompiler();
223 
224 		m_defaultArchitecture = m_config.defaultArchitecture;
225 	}
226 
227 	version(Windows) 
228 	private void migrateRepositoryFromRoaming(NativePath roamingDir, NativePath localDir)
229 	{
230         immutable roamingDirPath = roamingDir.toNativeString();
231 	    if (!existsDirectory(roamingDir)) return;
232         
233         immutable localDirPath = localDir.toNativeString();
234         logInfo("Detected a package cache in " ~ roamingDirPath ~ ". This will be migrated to " ~ localDirPath ~ ". Please wait...");
235         if (!existsDirectory(localDir))
236         {
237             mkdirRecurse(localDirPath);
238         }
239 
240         runCommand("xcopy /s /e /y " ~ roamingDirPath ~ " " ~ localDirPath ~ " > NUL"); 
241         rmdirRecurse(roamingDirPath);
242 	}
243 
244 
245 	@property void dryRun(bool v) { m_dryRun = v; }
246 
247 	/** Returns the root path (usually the current working directory).
248 	*/
249 	@property NativePath rootPath() const { return m_rootPath; }
250 	/// ditto
251 	@property void rootPath(NativePath root_path)
252 	{
253 		m_rootPath = root_path;
254 		if (!m_rootPath.absolute) m_rootPath = NativePath(getcwd()) ~ m_rootPath;
255 	}
256 
257 	/// Returns the name listed in the dub.json of the current
258 	/// application.
259 	@property string projectName() const { return m_project.name; }
260 
261 	@property NativePath projectPath() const { return m_projectPath; }
262 
263 	@property string[] configurations() const { return m_project.configurations; }
264 
265 	@property inout(PackageManager) packageManager() inout { return m_packageManager; }
266 
267 	@property inout(Project) project() inout { return m_project; }
268 
269 	/** Returns the default compiler binary to use for building D code.
270 
271 		If set, the "defaultCompiler" field of the DUB user or system
272 		configuration file will be used. Otherwise the PATH environment variable
273 		will be searched for files named "dmd", "gdc", "gdmd", "ldc2", "ldmd2"
274 		(in that order, taking into account operating system specific file
275 		extensions) and the first match is returned. If no match is found, "dmd"
276 		will be used.
277 	*/
278 	@property string defaultCompiler() const { return m_defaultCompiler; }
279 
280 	/** Returns the default architecture to use for building D code.
281 
282 		If set, the "defaultArchitecture" field of the DUB user or system
283 		configuration file will be used. Otherwise null will be returned.
284 	*/
285 	@property string defaultArchitecture() const { return m_defaultArchitecture; }
286 
287 	/** Loads the package that resides within the configured `rootPath`.
288 	*/
289 	void loadPackage()
290 	{
291 		loadPackage(m_rootPath);
292 	}
293 
294 	/// Loads the package from the specified path as the main project package.
295 	void loadPackage(NativePath path)
296 	{
297 		m_projectPath = path;
298 		updatePackageSearchPath();
299 		m_project = new Project(m_packageManager, m_projectPath);
300 	}
301 
302 	/// Loads a specific package as the main project package (can be a sub package)
303 	void loadPackage(Package pack)
304 	{
305 		m_projectPath = pack.path;
306 		updatePackageSearchPath();
307 		m_project = new Project(m_packageManager, pack);
308 	}
309 
310 	/** Loads a single file package.
311 
312 		Single-file packages are D files that contain a package receipe comment
313 		at their top. A recipe comment must be a nested `/+ ... +/` style
314 		comment, containing the virtual recipe file name and a colon, followed by the
315 		recipe contents (what would normally be in dub.sdl/dub.json).
316 
317 		Example:
318 		---
319 		/+ dub.sdl:
320 		   name "test"
321 		   dependency "vibe-d" version="~>0.7.29"
322 		+/
323 		import vibe.http.server;
324 
325 		void main()
326 		{
327 			auto settings = new HTTPServerSettings;
328 			settings.port = 8080;
329 			listenHTTP(settings, &hello);
330 		}
331 
332 		void hello(HTTPServerRequest req, HTTPServerResponse res)
333 		{
334 			res.writeBody("Hello, World!");
335 		}
336 		---
337 
338 		The script above can be invoked with "dub --single test.d".
339 	*/
340 	void loadSingleFilePackage(NativePath path)
341 	{
342 		import dub.recipe.io : parsePackageRecipe;
343 		import std.file : mkdirRecurse, readText;
344 		import std.path : baseName, stripExtension;
345 
346 		path = makeAbsolute(path);
347 
348 		string file_content = readText(path.toNativeString());
349 
350 		if (file_content.startsWith("#!")) {
351 			auto idx = file_content.indexOf('\n');
352 			enforce(idx > 0, "The source fine doesn't contain anything but a shebang line.");
353 			file_content = file_content[idx+1 .. $];
354 		}
355 
356 		file_content = file_content.strip();
357 
358 		string recipe_content;
359 
360 		if (file_content.startsWith("/+")) {
361 			file_content = file_content[2 .. $];
362 			auto idx = file_content.indexOf("+/");
363 			enforce(idx >= 0, "Missing \"+/\" to close comment.");
364 			recipe_content = file_content[0 .. idx].strip();
365 		} else throw new Exception("The source file must start with a recipe comment.");
366 
367 		auto nidx = recipe_content.indexOf('\n');
368 
369 		auto idx = recipe_content.indexOf(':');
370 		enforce(idx > 0 && (nidx < 0 || nidx > idx),
371 			"The first line of the recipe comment must list the recipe file name followed by a colon (e.g. \"/+ dub.sdl:\").");
372 		auto recipe_filename = recipe_content[0 .. idx];
373 		recipe_content = recipe_content[idx+1 .. $];
374 		auto recipe_default_package_name = path.toString.baseName.stripExtension.strip;
375 
376 		auto recipe = parsePackageRecipe(recipe_content, recipe_filename, null, recipe_default_package_name);
377 		enforce(recipe.buildSettings.sourceFiles.length == 0, "Single-file packages are not allowed to specify source files.");
378 		enforce(recipe.buildSettings.sourcePaths.length == 0, "Single-file packages are not allowed to specify source paths.");
379 		enforce(recipe.buildSettings.importPaths.length == 0, "Single-file packages are not allowed to specify import paths.");
380 		recipe.buildSettings.sourceFiles[""] = [path.toNativeString()];
381 		recipe.buildSettings.sourcePaths[""] = [];
382 		recipe.buildSettings.importPaths[""] = [];
383 		recipe.buildSettings.mainSourceFile = path.toNativeString();
384 		if (recipe.buildSettings.targetType == TargetType.autodetect)
385 			recipe.buildSettings.targetType = TargetType.executable;
386 
387 		auto pack = new Package(recipe, path.parentPath, null, "~master");
388 		loadPackage(pack);
389 	}
390 	/// ditto
391 	void loadSingleFilePackage(string path)
392 	{
393 		loadSingleFilePackage(NativePath(path));
394 	}
395 
396 	/** Disables the default search paths and only searches a specific directory
397 		for packages.
398 	*/
399 	void overrideSearchPath(NativePath path)
400 	{
401 		if (!path.absolute) path = NativePath(getcwd()) ~ path;
402 		m_overrideSearchPath = path;
403 		updatePackageSearchPath();
404 	}
405 
406 	/** Gets the default configuration for a particular build platform.
407 
408 		This forwards to `Project.getDefaultConfiguration` and requires a
409 		project to be loaded.
410 	*/
411 	string getDefaultConfiguration(BuildPlatform platform, bool allow_non_library_configs = true) const { return m_project.getDefaultConfiguration(platform, allow_non_library_configs); }
412 
413 	/** Attempts to upgrade the dependency selection of the loaded project.
414 
415 		Params:
416 			options = Flags that control how the upgrade is carried out
417 			packages_to_upgrade = Optional list of packages. If this list
418 				contains one or more packages, only those packages will
419 				be upgraded. Otherwise, all packages will be upgraded at
420 				once.
421 	*/
422 	void upgrade(UpgradeOptions options, string[] packages_to_upgrade = null)
423 	{
424 		// clear non-existent version selections
425 		if (!(options & UpgradeOptions.upgrade)) {
426 			next_pack:
427 			foreach (p; m_project.selections.selectedPackages) {
428 				auto dep = m_project.selections.getSelectedVersion(p);
429 				if (!dep.path.empty) {
430 					auto path = dep.path;
431 					if (!path.absolute) path = this.rootPath ~ path;
432 					try if (m_packageManager.getOrLoadPackage(path)) continue;
433 					catch (Exception e) { logDebug("Failed to load path based selection: %s", e.toString().sanitize); }
434 				} else {
435 					if (m_packageManager.getPackage(p, dep.version_)) continue;
436 					foreach (ps; m_packageSuppliers) {
437 						try {
438 							auto versions = ps.getVersions(p);
439 							if (versions.canFind!(v => dep.matches(v)))
440 								continue next_pack;
441 						} catch (Exception e) {
442 							logWarn("Error querying versions for %s, %s: %s", p, ps.description, e.msg);
443 							logDebug("Full error: %s", e.toString().sanitize());
444 						}
445 					}
446 				}
447 
448 				logWarn("Selected package %s %s doesn't exist. Using latest matching version instead.", p, dep);
449 				m_project.selections.deselectVersion(p);
450 			}
451 		}
452 
453 		Dependency[string] versions;
454 		if ((options & UpgradeOptions.useCachedResult) && m_project.isUpgradeCacheUpToDate() && !packages_to_upgrade.length) {
455 			logDiagnostic("Using cached upgrade results...");
456 			versions = m_project.getUpgradeCache();
457 		} else {
458 			auto resolver = new DependencyVersionResolver(this, options);
459 			foreach (p; packages_to_upgrade)
460 				resolver.addPackageToUpgrade(p);
461 			versions = resolver.resolve(m_project.rootPackage, m_project.selections);
462 			if (options & UpgradeOptions.useCachedResult) {
463 				logDiagnostic("Caching upgrade results...");
464 				m_project.setUpgradeCache(versions);
465 			}
466 		}
467 
468 		if (options & UpgradeOptions.printUpgradesOnly) {
469 			bool any = false;
470 			string rootbasename = getBasePackageName(m_project.rootPackage.name);
471 
472 			foreach (p, ver; versions) {
473 				if (!ver.path.empty) continue;
474 
475 				auto basename = getBasePackageName(p);
476 				if (basename == rootbasename) continue;
477 
478 				if (!m_project.selections.hasSelectedVersion(basename)) {
479 					logInfo("Non-selected package %s is available with version %s.",
480 						basename, ver);
481 					any = true;
482 					continue;
483 				}
484 				auto sver = m_project.selections.getSelectedVersion(basename);
485 				if (!sver.path.empty) continue;
486 				if (ver.version_ <= sver.version_) continue;
487 				logInfo("Package %s can be upgraded from %s to %s.",
488 					basename, sver, ver);
489 				any = true;
490 			}
491 			if (any) logInfo("Use \"dub upgrade\" to perform those changes.");
492 			return;
493 		}
494 
495 		foreach (p; versions.byKey) {
496 			auto ver = versions[p]; // Workaround for DMD 2.070.0 AA issue (crashes in aaApply2 if iterating by key+value)
497 			assert(!p.canFind(":"), "Resolved packages contain a sub package!?: "~p);
498 			Package pack;
499 			if (!ver.path.empty) {
500 				try pack = m_packageManager.getOrLoadPackage(ver.path);
501 				catch (Exception e) {
502 					logDebug("Failed to load path based selection: %s", e.toString().sanitize);
503 					continue;
504 				}
505 			} else {
506 				pack = m_packageManager.getBestPackage(p, ver);
507 				if (pack && m_packageManager.isManagedPackage(pack)
508 					&& ver.version_.isBranch && (options & UpgradeOptions.upgrade) != 0)
509 				{
510 					// TODO: only re-install if there is actually a new commit available
511 					logInfo("Re-installing branch based dependency %s %s", p, ver.toString());
512 					m_packageManager.remove(pack);
513 					pack = null;
514 				}
515 			}
516 
517 			FetchOptions fetchOpts;
518 			fetchOpts |= (options & UpgradeOptions.preRelease) != 0 ? FetchOptions.usePrerelease : FetchOptions.none;
519 			if (!pack) fetch(p, ver, defaultPlacementLocation, fetchOpts, "getting selected version");
520 			if ((options & UpgradeOptions.select) && p != m_project.rootPackage.name) {
521 				if (ver.path.empty) m_project.selections.selectVersion(p, ver.version_);
522 				else {
523 					NativePath relpath = ver.path;
524 					if (relpath.absolute) relpath = relpath.relativeTo(m_project.rootPackage.path);
525 					m_project.selections.selectVersion(p, relpath);
526 				}
527 			}
528 		}
529 
530 		m_project.reinit();
531 
532 		if ((options & UpgradeOptions.select) && !(options & UpgradeOptions.noSaveSelections))
533 			m_project.saveSelections();
534 	}
535 
536 	/** Generate project files for a specified generator.
537 
538 		Any existing project files will be overridden.
539 	*/
540 	void generateProject(string ide, GeneratorSettings settings)
541 	{
542 		auto generator = createProjectGenerator(ide, m_project);
543 		if (m_dryRun) return; // TODO: pass m_dryRun to the generator
544 		generator.generate(settings);
545 	}
546 
547 	/** Executes tests on the current project.
548 
549 		Throws an exception, if unittests failed.
550 	*/
551 	void testProject(GeneratorSettings settings, string config, NativePath custom_main_file)
552 	{
553 		if (!custom_main_file.empty && !custom_main_file.absolute) custom_main_file = getWorkingDirectory() ~ custom_main_file;
554 
555 		if (config.length == 0) {
556 			// if a custom main file was given, favor the first library configuration, so that it can be applied
557 			if (!custom_main_file.empty) config = m_project.getDefaultConfiguration(settings.platform, false);
558 			// else look for a "unittest" configuration
559 			if (!config.length && m_project.rootPackage.configurations.canFind("unittest")) config = "unittest";
560 			// if not found, fall back to the first "library" configuration
561 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, false);
562 			// if still nothing found, use the first executable configuration
563 			if (!config.length) config = m_project.getDefaultConfiguration(settings.platform, true);
564 		}
565 
566 		auto generator = createProjectGenerator("build", m_project);
567 
568 		auto test_config = format("%s-test-%s", m_project.rootPackage.name.replace(".", "-").replace(":", "-"), config);
569 
570 		BuildSettings lbuildsettings = settings.buildSettings;
571 		m_project.addBuildSettings(lbuildsettings, settings.platform, config, null, true);
572 		if (lbuildsettings.targetType == TargetType.none) {
573 			logInfo(`Configuration '%s' has target type "none". Skipping test.`, config);
574 			return;
575 		}
576 
577 		if (lbuildsettings.targetType == TargetType.executable && config == "unittest") {
578 			logInfo("Running custom 'unittest' configuration.", config);
579 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
580 			settings.config = config;
581 		} else if (lbuildsettings.sourceFiles.empty) {
582 			logInfo(`No source files found in configuration '%s'. Falling back to "dub -b unittest".`, config);
583 			if (!custom_main_file.empty) logWarn("Ignoring custom main file.");
584 			settings.config = m_project.getDefaultConfiguration(settings.platform);
585 		} else {
586 			import std.algorithm : remove;
587 
588 			logInfo(`Generating test runner configuration '%s' for '%s' (%s).`, test_config, config, lbuildsettings.targetType);
589 
590 			BuildSettingsTemplate tcinfo = m_project.rootPackage.recipe.getConfiguration(config).buildSettings;
591 			tcinfo.targetType = TargetType.executable;
592 			tcinfo.targetName = test_config;
593 			// HACK for vibe.d's legacy main() behavior:
594 			tcinfo.versions[""] ~= "VibeCustomMain";
595 			m_project.rootPackage.recipe.buildSettings.versions[""] = m_project.rootPackage.recipe.buildSettings.versions.get("", null).remove!(v => v == "VibeDefaultMain");
596 			// TODO: remove this ^ once vibe.d has removed the default main implementation
597 
598 			auto mainfil = tcinfo.mainSourceFile;
599 			if (!mainfil.length) mainfil = m_project.rootPackage.recipe.buildSettings.mainSourceFile;
600 
601 			string custommodname;
602 			if (!custom_main_file.empty) {
603 				import std.path;
604 				tcinfo.sourceFiles[""] ~= custom_main_file.relativeTo(m_project.rootPackage.path).toNativeString();
605 				tcinfo.importPaths[""] ~= custom_main_file.parentPath.toNativeString();
606 				custommodname = custom_main_file.head.toString().baseName(".d");
607 			}
608 
609 			// prepare the list of tested modules
610 			string[] import_modules;
611 			foreach (file; lbuildsettings.sourceFiles) {
612 				if (file.endsWith(".d")) {
613 					auto fname = NativePath(file).head.toString();
614 					if (NativePath(file).relativeTo(m_project.rootPackage.path) == NativePath(mainfil)) {
615 						logWarn("Excluding main source file %s from test.", mainfil);
616 						tcinfo.excludedSourceFiles[""] ~= mainfil;
617 						continue;
618 					}
619 					if (fname == "package.d") {
620 						logWarn("Excluding package.d file from test due to https://issues.dlang.org/show_bug.cgi?id=11847");
621 						continue;
622 					}
623 					import_modules ~= dub.internal.utils.determineModuleName(lbuildsettings, NativePath(file), m_project.rootPackage.path);
624 				}
625 			}
626 
627 			// generate main file
628 			NativePath mainfile = getTempFile("dub_test_root", ".d");
629 			tcinfo.sourceFiles[""] ~= mainfile.toNativeString();
630 			tcinfo.mainSourceFile = mainfile.toNativeString();
631 			if (!m_dryRun) {
632 				auto fil = openFile(mainfile, FileMode.createTrunc);
633 				scope(exit) fil.close();
634 				fil.write("module dub_test_root;\n");
635 				fil.write("import std.typetuple;\n");
636 				foreach (mod; import_modules) fil.write(format("static import %s;\n", mod));
637 				fil.write("alias allModules = TypeTuple!(");
638 				foreach (i, mod; import_modules) {
639 					if (i > 0) fil.write(", ");
640 					fil.write(mod);
641 				}
642 				fil.write(");\n");
643 				if (custommodname.length) {
644 					fil.write(format("import %s;\n", custommodname));
645 				} else {
646 					fil.write(q{
647 						import std.stdio;
648 						import core.runtime;
649 
650 						void main() { writeln("All unit tests have been run successfully."); }
651 						shared static this() {
652 							version (Have_tested) {
653 								import tested;
654 								import core.runtime;
655 								import std.exception;
656 								Runtime.moduleUnitTester = () => true;
657 								//runUnitTests!app(new JsonTestResultWriter("results.json"));
658 								enforce(runUnitTests!allModules(new ConsoleTestResultWriter), "Unit tests failed.");
659 							}
660 						}
661 					});
662 				}
663 			}
664 			m_project.rootPackage.recipe.configurations ~= ConfigurationInfo(test_config, tcinfo);
665 			m_project = new Project(m_packageManager, m_project.rootPackage);
666 
667 			settings.config = test_config;
668 		}
669 
670 		generator.generate(settings);
671 	}
672 
673 	/** Prints the specified build settings necessary for building the root package.
674 	*/
675 	void listProjectData(GeneratorSettings settings, string[] requestedData, ListBuildSettingsFormat list_type)
676 	{
677 		import std.stdio;
678 		import std.ascii : newline;
679 
680 		// Split comma-separated lists
681 		string[] requestedDataSplit =
682 			requestedData
683 			.map!(a => a.splitter(",").map!strip)
684 			.joiner()
685 			.array();
686 
687 		auto data = m_project.listBuildSettings(settings, requestedDataSplit, list_type);
688 
689 		string delimiter;
690 		final switch (list_type) with (ListBuildSettingsFormat) {
691 			case list: delimiter = newline ~ newline; break;
692 			case listNul: delimiter = "\0\0"; break;
693 			case commandLine: delimiter = " "; break;
694 			case commandLineNul: delimiter = "\0\0"; break;
695 		}
696 
697 		write(data.joiner(delimiter));
698 		if (delimiter != "\0\0") writeln();
699 	}
700 
701 	/// Cleans intermediate/cache files of the given package
702 	void cleanPackage(NativePath path)
703 	{
704 		logInfo("Cleaning package at %s...", path.toNativeString());
705 		enforce(!Package.findPackageFile(path).empty, "No package found.", path.toNativeString());
706 
707 		// TODO: clear target files and copy files
708 
709 		if (existsFile(path ~ ".dub/build")) rmdirRecurse((path ~ ".dub/build").toNativeString());
710 		if (existsFile(path ~ ".dub/obj")) rmdirRecurse((path ~ ".dub/obj").toNativeString());
711 	}
712 
713 	/// Fetches the package matching the dependency and places it in the specified location.
714 	Package fetch(string packageId, const Dependency dep, PlacementLocation location, FetchOptions options, string reason = "")
715 	{
716 		Json pinfo;
717 		PackageSupplier supplier;
718 		foreach(ps; m_packageSuppliers){
719 			try {
720 				pinfo = ps.fetchPackageRecipe(packageId, dep, (options & FetchOptions.usePrerelease) != 0);
721 				if (pinfo.type == Json.Type.null_)
722 					continue;
723 				supplier = ps;
724 				break;
725 			} catch(Exception e) {
726 				logWarn("Package %s not found for %s: %s", packageId, ps.description, e.msg);
727 				logDebug("Full error: %s", e.toString().sanitize());
728 			}
729 		}
730 		enforce(pinfo.type != Json.Type.undefined, "No package "~packageId~" was found matching the dependency "~dep.toString());
731 		string ver = pinfo["version"].get!string;
732 
733 		NativePath placement;
734 		final switch (location) {
735 			case PlacementLocation.local: placement = m_rootPath; break;
736 			case PlacementLocation.user: placement = m_dirs.localRepository ~ "packages/"; break;
737 			case PlacementLocation.system: placement = m_dirs.systemSettings ~ "packages/"; break;
738 		}
739 
740 		// always upgrade branch based versions - TODO: actually check if there is a new commit available
741 		Package existing;
742 		try existing = m_packageManager.getPackage(packageId, ver, placement);
743 		catch (Exception e) {
744 			logWarn("Failed to load existing package %s: %s", ver, e.msg);
745 			logDiagnostic("Full error: %s", e.toString().sanitize);
746 		}
747 
748 		if (options & FetchOptions.printOnly) {
749 			if (existing && existing.version_ != Version(ver))
750 				logInfo("A new version for %s is available (%s -> %s). Run \"dub upgrade %s\" to switch.",
751 					packageId, existing.version_, ver, packageId);
752 			return null;
753 		}
754 
755 		if (existing) {
756 			if (!ver.startsWith("~") || !(options & FetchOptions.forceBranchUpgrade) || location == PlacementLocation.local) {
757 				// TODO: support git working trees by performing a "git pull" instead of this
758 				logDiagnostic("Package %s %s (%s) is already present with the latest version, skipping upgrade.",
759 					packageId, ver, placement);
760 				return existing;
761 			} else {
762 				logInfo("Removing %s %s to prepare replacement with a new version.", packageId, ver);
763 				if (!m_dryRun) m_packageManager.remove(existing);
764 			}
765 		}
766 
767 		if (reason.length) logInfo("Fetching %s %s (%s)...", packageId, ver, reason);
768 		else logInfo("Fetching %s %s...", packageId, ver);
769 		if (m_dryRun) return null;
770 
771 		logDebug("Acquiring package zip file");
772 
773 		auto clean_package_version = ver[ver.startsWith("~") ? 1 : 0 .. $];
774 		clean_package_version = clean_package_version.replace("+", "_"); // + has special meaning for Optlink
775 		if (!placement.existsFile())
776 			mkdirRecurse(placement.toNativeString());
777 		NativePath dstpath = placement ~ (packageId ~ "-" ~ clean_package_version);
778 		if (!dstpath.existsFile())
779 			mkdirRecurse(dstpath.toNativeString());
780 
781 		// Support libraries typically used with git submodules like ae.
782 		// Such libraries need to have ".." as import path but this can create
783 		// import path leakage.
784 		dstpath = dstpath ~ packageId;
785 
786 		import std.datetime : seconds;
787 		auto lock = lockFile(dstpath.toNativeString() ~ ".lock", 30.seconds); // possibly wait for other dub instance
788 		if (dstpath.existsFile())
789 		{
790 			m_packageManager.refresh(false);
791 			return m_packageManager.getPackage(packageId, ver, dstpath);
792 		}
793 
794 		// repeat download on corrupted zips, see #1336
795 		foreach_reverse (i; 0..3)
796 		{
797 			import std.zip : ZipException;
798 
799 			auto path = getTempFile(packageId, ".zip");
800 			supplier.fetchPackage(path, packageId, dep, (options & FetchOptions.usePrerelease) != 0); // Q: continue on fail?
801 			scope(exit) std.file.remove(path.toNativeString());
802 			logDiagnostic("Placing to %s...", placement.toNativeString());
803 
804 			try {
805 				return m_packageManager.storeFetchedPackage(path, pinfo, dstpath);
806 			} catch (ZipException e) {
807 				logInfo("Failed to extract zip archive for %s %s...", packageId, ver);
808 				// rethrow the exception at the end of the loop
809 				if (i == 0)
810 					throw e;
811 			}
812 		}
813 		assert(0, "Should throw a ZipException instead.");
814 	}
815 
816 	/** Removes a specific locally cached package.
817 
818 		This will delete the package files from disk and removes the
819 		corresponding entry from the list of known packages.
820 
821 		Params:
822 			pack = Package instance to remove
823 	*/
824 	void remove(in Package pack)
825 	{
826 		logInfo("Removing %s in %s", pack.name, pack.path.toNativeString());
827 		if (!m_dryRun) m_packageManager.remove(pack);
828 	}
829 
830 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
831 	void remove(in Package pack, bool force_remove)
832 	{
833 		remove(pack);
834 	}
835 
836 	/// @see remove(string, string, RemoveLocation)
837 	enum RemoveVersionWildcard = "*";
838 
839 	/** Removes one or more versions of a locally cached package.
840 
841 		This will remove a given package with a specified version from the
842 		given location. It will remove at most one package, unless `version_`
843 		is set to `RemoveVersionWildcard`.
844 
845 		Params:
846 			package_id = Name of the package to be removed
847 			location_ = Specifies the location to look for the given package
848 				name/version.
849 			resolve_version = Callback to select package version.
850 	*/
851 	void remove(string package_id, PlacementLocation location,
852 				scope size_t delegate(in Package[] packages) resolve_version)
853 	{
854 		enforce(!package_id.empty);
855 		if (location == PlacementLocation.local) {
856 			logInfo("To remove a locally placed package, make sure you don't have any data"
857 					~ "\nleft in it's directory and then simply remove the whole directory.");
858 			throw new Exception("dub cannot remove locally installed packages.");
859 		}
860 
861 		Package[] packages;
862 
863 		// Retrieve packages to be removed.
864 		foreach(pack; m_packageManager.getPackageIterator(package_id))
865 			if (m_packageManager.isManagedPackage(pack))
866 				packages ~= pack;
867 
868 		// Check validity of packages to be removed.
869 		if(packages.empty) {
870 			throw new Exception("Cannot find package to remove. ("
871 				~ "id: '" ~ package_id ~ "', location: '" ~ to!string(location) ~ "'"
872 				~ ")");
873 		}
874 
875 		// Sort package list in ascending version order
876 		packages.sort!((a, b) => a.version_ < b.version_);
877 
878 		immutable idx = resolve_version(packages);
879 		if (idx == size_t.max)
880 			return;
881 		else if (idx != packages.length)
882 			packages = packages[idx .. idx + 1];
883 
884 		logDebug("Removing %s packages.", packages.length);
885 		foreach(pack; packages) {
886 			try {
887 				remove(pack);
888 				logInfo("Removed %s, version %s.", package_id, pack.version_);
889 			} catch (Exception e) {
890 				logError("Failed to remove %s %s: %s", package_id, pack.version_, e.msg);
891 				logInfo("Continuing with other packages (if any).");
892 			}
893 		}
894 	}
895 
896 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
897 	void remove(string package_id, PlacementLocation location, bool force_remove,
898 				scope size_t delegate(in Package[] packages) resolve_version)
899 	{
900 		remove(package_id, location, resolve_version);
901 	}
902 
903 	/** Removes a specific version of a package.
904 
905 		Params:
906 			package_id = Name of the package to be removed
907 			version_ = Identifying a version or a wild card. If an empty string
908 				is passed, the package will be removed from the location, if
909 				there is only one version retrieved. This will throw an
910 				exception, if there are multiple versions retrieved.
911 			location_ = Specifies the location to look for the given package
912 				name/version.
913 	 */
914 	void remove(string package_id, string version_, PlacementLocation location)
915 	{
916 		remove(package_id, location, (in packages) {
917 			if (version_ == RemoveVersionWildcard)
918 				return packages.length;
919 			if (version_.empty && packages.length > 1) {
920 				logError("Cannot remove package '" ~ package_id ~ "', there are multiple possibilities at location\n"
921 						 ~ "'" ~ to!string(location) ~ "'.");
922 				logError("Available versions:");
923 				foreach(pack; packages)
924 					logError("  %s", pack.version_);
925 				throw new Exception("Please specify a individual version using --version=... or use the"
926 									~ " wildcard --version=" ~ RemoveVersionWildcard ~ " to remove all versions.");
927 			}
928 			foreach (i, p; packages) {
929 				if (p.version_ == Version(version_))
930 					return i;
931 			}
932 			throw new Exception("Cannot find package to remove. ("
933 				~ "id: '" ~ package_id ~ "', version: '" ~ version_ ~ "', location: '" ~ to!string(location) ~ "'"
934 				~ ")");
935 		});
936 	}
937 
938 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
939 	void remove(string package_id, string version_, PlacementLocation location, bool force_remove)
940 	{
941 		remove(package_id, version_, location);
942 	}
943 
944 	/** Adds a directory to the list of locally known packages.
945 
946 		Forwards to `PackageManager.addLocalPackage`.
947 
948 		Params:
949 			path = Path to the package
950 			ver = Optional version to associate with the package (can be left
951 				empty)
952 			system = Make the package known system wide instead of user wide
953 				(requires administrator privileges).
954 
955 		See_Also: `removeLocalPackage`
956 	*/
957 	void addLocalPackage(string path, string ver, bool system)
958 	{
959 		if (m_dryRun) return;
960 		m_packageManager.addLocalPackage(makeAbsolute(path), ver, system ? LocalPackageType.system : LocalPackageType.user);
961 	}
962 
963 	/** Removes a directory from the list of locally known packages.
964 
965 		Forwards to `PackageManager.removeLocalPackage`.
966 
967 		Params:
968 			path = Path to the package
969 			system = Make the package known system wide instead of user wide
970 				(requires administrator privileges).
971 
972 		See_Also: `addLocalPackage`
973 	*/
974 	void removeLocalPackage(string path, bool system)
975 	{
976 		if (m_dryRun) return;
977 		m_packageManager.removeLocalPackage(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
978 	}
979 
980 	/** Registers a local directory to search for packages to use for satisfying
981 		dependencies.
982 
983 		Params:
984 			path = Path to a directory containing package directories
985 			system = Make the package known system wide instead of user wide
986 				(requires administrator privileges).
987 
988 		See_Also: `removeSearchPath`
989 	*/
990 	void addSearchPath(string path, bool system)
991 	{
992 		if (m_dryRun) return;
993 		m_packageManager.addSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
994 	}
995 
996 	/** Unregisters a local directory search path.
997 
998 		Params:
999 			path = Path to a directory containing package directories
1000 			system = Make the package known system wide instead of user wide
1001 				(requires administrator privileges).
1002 
1003 		See_Also: `addSearchPath`
1004 	*/
1005 	void removeSearchPath(string path, bool system)
1006 	{
1007 		if (m_dryRun) return;
1008 		m_packageManager.removeSearchPath(makeAbsolute(path), system ? LocalPackageType.system : LocalPackageType.user);
1009 	}
1010 
1011 	/** Queries all package suppliers with the given query string.
1012 
1013 		Returns a list of tuples, where the first entry is the human readable
1014 		name of the package supplier and the second entry is the list of
1015 		matched packages.
1016 
1017 		See_Also: `PackageSupplier.searchPackages`
1018 	*/
1019 	auto searchPackages(string query)
1020 	{
1021 		import std.typecons : Tuple, tuple;
1022 		Tuple!(string, PackageSupplier.SearchResult[])[] results;
1023 		foreach (ps; this.m_packageSuppliers) {
1024 			try
1025 				results ~= tuple(ps.description, ps.searchPackages(query));
1026 			catch (Exception e) {
1027 				logWarn("Searching %s for '%s' failed: %s", ps.description, query, e.msg);
1028 			}
1029 		}
1030 		return results.filter!(tup => tup[1].length);
1031 	}
1032 
1033 	/** Returns a list of all available versions (including branches) for a
1034 		particular package.
1035 
1036 		The list returned is based on the registered package suppliers. Local
1037 		packages are not queried in the search for versions.
1038 
1039 		See_also: `getLatestVersion`
1040 	*/
1041 	Version[] listPackageVersions(string name)
1042 	{
1043 		Version[] versions;
1044 		foreach (ps; this.m_packageSuppliers) {
1045 			try versions ~= ps.getVersions(name);
1046 			catch (Exception e) {
1047 				logWarn("Failed to get versions for package %s on provider %s: %s", name, ps.description, e.msg);
1048 			}
1049 		}
1050 		return versions.sort().uniq.array;
1051 	}
1052 
1053 	/** Returns the latest available version for a particular package.
1054 
1055 		This function returns the latest numbered version of a package. If no
1056 		numbered versions are available, it will return an available branch,
1057 		preferring "~master".
1058 
1059 		Params:
1060 			package_name: The name of the package in question.
1061 			prefer_stable: If set to `true` (the default), returns the latest
1062 				stable version, even if there are newer pre-release versions.
1063 
1064 		See_also: `listPackageVersions`
1065 	*/
1066 	Version getLatestVersion(string package_name, bool prefer_stable = true)
1067 	{
1068 		auto vers = listPackageVersions(package_name);
1069 		enforce(!vers.empty, "Failed to find any valid versions for a package name of '"~package_name~"'.");
1070 		auto final_versions = vers.filter!(v => !v.isBranch && !v.isPreRelease).array;
1071 		if (prefer_stable && final_versions.length) return final_versions[$-1];
1072 		else if (vers[$-1].isBranch) return vers[$-1];
1073 		else return vers[$-1];
1074 	}
1075 
1076 	/** Initializes a directory with a package skeleton.
1077 
1078 		Params:
1079 			path = Path of the directory to create the new package in. The
1080 				directory will be created if it doesn't exist.
1081 			deps = List of dependencies to add to the package recipe.
1082 			type = Specifies the type of the application skeleton to use.
1083 			format = Determines the package recipe format to use.
1084 			recipe_callback = Optional callback that can be used to
1085 				customize the recipe before it gets written.
1086 	*/
1087 	void createEmptyPackage(NativePath path, string[] deps, string type,
1088 		PackageFormat format = PackageFormat.sdl,
1089 		scope void delegate(ref PackageRecipe, ref PackageFormat) recipe_callback = null)
1090 	{
1091 		if (!path.absolute) path = m_rootPath ~ path;
1092 		path.normalize();
1093 
1094 		string[string] depVers;
1095 		string[] notFound; // keep track of any failed packages in here
1096 		foreach (dep; deps) {
1097 			Version ver;
1098 			try {
1099 				ver = getLatestVersion(dep);
1100 				depVers[dep] = ver.isBranch ? ver.toString() : "~>" ~ ver.toString();
1101 			} catch (Exception e) {
1102 				notFound ~= dep;
1103 			}
1104 		}
1105 
1106 		if(notFound.length > 1){
1107 			throw new Exception(.format("Couldn't find packages: %-(%s, %).", notFound));
1108 		}
1109 		else if(notFound.length == 1){
1110 			throw new Exception(.format("Couldn't find package: %-(%s, %).", notFound));
1111 		}
1112 
1113 		if (m_dryRun) return;
1114 
1115 		initPackage(path, depVers, type, format, recipe_callback);
1116 
1117 		//Act smug to the user.
1118 		logInfo("Successfully created an empty project in '%s'.", path.toNativeString());
1119 	}
1120 
1121 	/** Converts the package recipe of the loaded root package to the given format.
1122 
1123 		Params:
1124 			destination_file_ext = The file extension matching the desired
1125 				format. Possible values are "json" or "sdl".
1126 			print_only = Print the converted recipe instead of writing to disk
1127 	*/
1128 	void convertRecipe(string destination_file_ext, bool print_only = false)
1129 	{
1130 		import std.path : extension;
1131 		import std.stdio : stdout;
1132 		import dub.recipe.io : serializePackageRecipe, writePackageRecipe;
1133 
1134 		if (print_only) {
1135 			auto dst = stdout.lockingTextWriter;
1136 			serializePackageRecipe(dst, m_project.rootPackage.rawRecipe, "dub."~destination_file_ext);
1137 			return;
1138 		}
1139 
1140 		auto srcfile = m_project.rootPackage.recipePath;
1141 		auto srcext = srcfile.head.toString().extension;
1142 		if (srcext == "."~destination_file_ext) {
1143 			logInfo("Package format is already %s.", destination_file_ext);
1144 			return;
1145 		}
1146 
1147 		writePackageRecipe(srcfile.parentPath ~ ("dub."~destination_file_ext), m_project.rootPackage.rawRecipe);
1148 		removeFile(srcfile);
1149 	}
1150 
1151 	/** Runs DDOX to generate or serve documentation.
1152 
1153 		Params:
1154 			run = If set to true, serves documentation on a local web server.
1155 				Otherwise generates actual HTML files.
1156 			generate_args = Additional command line arguments to pass to
1157 				"ddox generate-html" or "ddox serve-html".
1158 	*/
1159 	void runDdox(bool run, string[] generate_args = null)
1160 	{
1161 		import std.process : browse;
1162 
1163 		if (m_dryRun) return;
1164 
1165 		// allow to choose a custom ddox tool
1166 		auto tool = m_project.rootPackage.recipe.ddoxTool;
1167 		if (tool.empty) tool = "ddox";
1168 
1169 		auto tool_pack = m_packageManager.getBestPackage(tool, ">=0.0.0");
1170 		if (!tool_pack) tool_pack = m_packageManager.getBestPackage(tool, "~master");
1171 		if (!tool_pack) {
1172 			logInfo("%s is not present, getting and storing it user wide", tool);
1173 			tool_pack = fetch(tool, Dependency(">=0.0.0"), defaultPlacementLocation, FetchOptions.none);
1174 		}
1175 
1176 		auto ddox_dub = new Dub(null, m_packageSuppliers);
1177 		ddox_dub.loadPackage(tool_pack.path);
1178 		ddox_dub.upgrade(UpgradeOptions.select);
1179 
1180 		auto compiler_binary = this.defaultCompiler;
1181 
1182 		GeneratorSettings settings;
1183 		settings.config = "application";
1184 		settings.compiler = getCompiler(compiler_binary); // TODO: not using --compiler ???
1185 		settings.platform = settings.compiler.determinePlatform(settings.buildSettings, compiler_binary);
1186 		settings.buildType = "debug";
1187 		settings.run = true;
1188 
1189 		auto filterargs = m_project.rootPackage.recipe.ddoxFilterArgs.dup;
1190 		if (filterargs.empty) filterargs = ["--min-protection=Protected", "--only-documented"];
1191 
1192 		settings.runArgs = "filter" ~ filterargs ~ "docs.json";
1193 		ddox_dub.generateProject("build", settings);
1194 
1195 		auto p = tool_pack.path;
1196 		p.endsWithSlash = true;
1197 		auto tool_path = p.toNativeString();
1198 
1199 		if (run) {
1200 			settings.runArgs = ["serve-html", "--navigation-type=ModuleTree", "docs.json", "--web-file-dir="~tool_path~"public"] ~ generate_args;
1201 			browse("http://127.0.0.1:8080/");
1202 		} else {
1203 			settings.runArgs = ["generate-html", "--navigation-type=ModuleTree", "docs.json", "docs"] ~ generate_args;
1204 		}
1205 		ddox_dub.generateProject("build", settings);
1206 
1207 		if (!run) {
1208 			// TODO: ddox should copy those files itself
1209 			version(Windows) runCommand("xcopy /S /D "~tool_path~"public\\* docs\\");
1210 			else runCommand("rsync -ru '"~tool_path~"public/' docs/");
1211 		}
1212 	}
1213 
1214 	private void updatePackageSearchPath()
1215 	{
1216 		if (!m_overrideSearchPath.empty) {
1217 			m_packageManager.disableDefaultSearchPaths = true;
1218 			m_packageManager.searchPath = [m_overrideSearchPath];
1219 		} else {
1220 			auto p = environment.get("DUBPATH");
1221 			NativePath[] paths;
1222 
1223 			version(Windows) enum pathsep = ";";
1224 			else enum pathsep = ":";
1225 			if (p.length) paths ~= p.split(pathsep).map!(p => NativePath(p))().array();
1226 			m_packageManager.disableDefaultSearchPaths = false;
1227 			m_packageManager.searchPath = paths;
1228 		}
1229 	}
1230 
1231 	private void determineDefaultCompiler()
1232 	{
1233 		import std.file : thisExePath;
1234 		import std.path : buildPath, dirName, expandTilde, isAbsolute, isDirSeparator;
1235 		import std.process : environment;
1236 		import std.range : front;
1237 		import std.regex : ctRegex, matchFirst;
1238 
1239 		m_defaultCompiler = m_config.defaultCompiler.expandTilde;
1240 		if (m_defaultCompiler.length && m_defaultCompiler.isAbsolute)
1241 			return;
1242 
1243 		auto dubPrefix = m_defaultCompiler.matchFirst(ctRegex!(`^\$DUB_BINARY_PATH`));
1244 		if(!dubPrefix.empty)
1245 		{
1246 			m_defaultCompiler = thisExePath().dirName() ~ dubPrefix.post;
1247 			return;
1248 		}
1249 
1250 		if (!find!isDirSeparator(m_defaultCompiler).empty)
1251 			throw new Exception("defaultCompiler specified in a DUB config file cannot use an unqualified relative path:\n\n" ~ m_defaultCompiler ~
1252 			"\n\nUse \"$DUB_BINARY_PATH/../path/you/want\" instead.");
1253 
1254 		version (Windows) enum sep = ";", exe = ".exe";
1255 		version (Posix) enum sep = ":", exe = "";
1256 
1257 		auto compilers = ["dmd", "gdc", "gdmd", "ldc2", "ldmd2"];
1258 		// If a compiler name is specified, look for it next to dub.
1259 		// Otherwise, look for any of the common compilers adjacent to dub.
1260 		if (m_defaultCompiler.length)
1261 		{
1262 			string compilerPath = buildPath(thisExePath().dirName(), m_defaultCompiler ~ exe);
1263 			if (existsFile(compilerPath))
1264 			{
1265 				m_defaultCompiler = compilerPath;
1266 				return;
1267 			}
1268 		}
1269 		else
1270 		{
1271 			auto nextFound = compilers.find!(bin => existsFile(buildPath(thisExePath().dirName(), bin ~ exe)));
1272 			if (!nextFound.empty)
1273 			{
1274 				m_defaultCompiler = buildPath(thisExePath().dirName(),  nextFound.front ~ exe);
1275 				return;
1276 			}
1277 		}
1278 
1279 		// If nothing found next to dub, search the user's PATH, starting
1280 		// with the compiler name from their DUB config file, if specified.
1281 		if (m_defaultCompiler.length)
1282 			compilers = m_defaultCompiler ~ compilers;
1283 		auto paths = environment.get("PATH", "").splitter(sep).map!NativePath;
1284 		auto res = compilers.find!(bin => paths.canFind!(p => existsFile(p ~ (bin~exe))));
1285 		m_defaultCompiler = res.empty ? compilers[0] : res.front;
1286 	}
1287 
1288 	private NativePath makeAbsolute(NativePath p) const { return p.absolute ? p : m_rootPath ~ p; }
1289 	private NativePath makeAbsolute(string p) const { return makeAbsolute(NativePath(p)); }
1290 }
1291 
1292 
1293 /// Option flags for `Dub.fetch`
1294 enum FetchOptions
1295 {
1296 	none = 0,
1297 	forceBranchUpgrade = 1<<0,
1298 	usePrerelease = 1<<1,
1299 	forceRemove = 1<<2, /// Deprecated, does nothing.
1300 	printOnly = 1<<3,
1301 }
1302 
1303 /// Option flags for `Dub.upgrade`
1304 enum UpgradeOptions
1305 {
1306 	none = 0,
1307 	upgrade = 1<<1, /// Upgrade existing packages
1308 	preRelease = 1<<2, /// inclde pre-release versions in upgrade
1309 	forceRemove = 1<<3, /// Deprecated, does nothing.
1310 	select = 1<<4, /// Update the dub.selections.json file with the upgraded versions
1311 	printUpgradesOnly = 1<<5, /// Instead of downloading new packages, just print a message to notify the user of their existence
1312 	useCachedResult = 1<<6, /// Use cached information stored with the package to determine upgrades
1313 	noSaveSelections = 1<<7, /// Don't store updated selections on disk
1314 }
1315 
1316 /// Determines which of the default package suppliers are queried for packages.
1317 enum SkipPackageSuppliers {
1318 	none,       /// Uses all configured package suppliers.
1319 	standard,   /// Does not use the default package suppliers (`defaultPackageSuppliers`).
1320 	configured, /// Does not use default suppliers or suppliers configured in DUB's configuration file
1321 	all         /// Uses only manually specified package suppliers.
1322 }
1323 
1324 private class DependencyVersionResolver : DependencyResolver!(Dependency, Dependency) {
1325 	protected {
1326 		Dub m_dub;
1327 		UpgradeOptions m_options;
1328 		Dependency[][string] m_packageVersions;
1329 		Package[string] m_remotePackages;
1330 		SelectedVersions m_selectedVersions;
1331 		Package m_rootPackage;
1332 		bool[string] m_packagesToUpgrade;
1333 		Package[PackageDependency] m_packages;
1334 		TreeNodes[][TreeNode] m_children;
1335 	}
1336 
1337 
1338 	this(Dub dub, UpgradeOptions options)
1339 	{
1340 		m_dub = dub;
1341 		m_options = options;
1342 	}
1343 
1344 	void addPackageToUpgrade(string name)
1345 	{
1346 		m_packagesToUpgrade[name] = true;
1347 	}
1348 
1349 	Dependency[string] resolve(Package root, SelectedVersions selected_versions)
1350 	{
1351 		m_rootPackage = root;
1352 		m_selectedVersions = selected_versions;
1353 		return super.resolve(TreeNode(root.name, Dependency(root.version_)), (m_options & UpgradeOptions.printUpgradesOnly) == 0);
1354 	}
1355 
1356 	protected bool isFixedPackage(string pack)
1357 	{
1358 		return m_packagesToUpgrade !is null && pack !in m_packagesToUpgrade;
1359 	}
1360 
1361 	protected override Dependency[] getAllConfigs(string pack)
1362 	{
1363 		if (auto pvers = pack in m_packageVersions)
1364 			return *pvers;
1365 
1366 		if ((!(m_options & UpgradeOptions.upgrade) || isFixedPackage(pack)) && m_selectedVersions.hasSelectedVersion(pack)) {
1367 			auto ret = [m_selectedVersions.getSelectedVersion(pack)];
1368 			logDiagnostic("Using fixed selection %s %s", pack, ret[0]);
1369 			m_packageVersions[pack] = ret;
1370 			return ret;
1371 		}
1372 
1373 		logDiagnostic("Search for versions of %s (%s package suppliers)", pack, m_dub.m_packageSuppliers.length);
1374 		Version[] versions;
1375 		foreach (p; m_dub.packageManager.getPackageIterator(pack))
1376 			versions ~= p.version_;
1377 
1378 		foreach (ps; m_dub.m_packageSuppliers) {
1379 			try {
1380 				auto vers = ps.getVersions(pack);
1381 				vers.reverse();
1382 				if (!vers.length) {
1383 					logDiagnostic("No versions for %s for %s", pack, ps.description);
1384 					continue;
1385 				}
1386 
1387 				versions ~= vers;
1388 				break;
1389 			} catch (Exception e) {
1390 				logWarn("Package %s not found in %s: %s", pack, ps.description, e.msg);
1391 				logDebug("Full error: %s", e.toString().sanitize);
1392 			}
1393 		}
1394 
1395 		// sort by version, descending, and remove duplicates
1396 		versions = versions.sort!"a>b".uniq.array;
1397 
1398 		// move pre-release versions to the back of the list if no preRelease flag is given
1399 		if (!(m_options & UpgradeOptions.preRelease))
1400 			versions = versions.filter!(v => !v.isPreRelease).array ~ versions.filter!(v => v.isPreRelease).array;
1401 
1402 		// filter out invalid/unreachable dependency specs
1403 		versions = versions.filter!((v) {
1404 				bool valid = getPackage(pack, Dependency(v)) !is null;
1405 				if (!valid) logDiagnostic("Excluding invalid dependency specification %s %s from dependency resolution process.", pack, v);
1406 				return valid;
1407 			}).array;
1408 
1409 		if (!versions.length) logDiagnostic("Nothing found for %s", pack);
1410 		else logDiagnostic("Return for %s: %s", pack, versions);
1411 
1412 		auto ret = versions.map!(v => Dependency(v)).array;
1413 		m_packageVersions[pack] = ret;
1414 		return ret;
1415 	}
1416 
1417 	protected override Dependency[] getSpecificConfigs(string pack, TreeNodes nodes)
1418 	{
1419 		if (!nodes.configs.path.empty && getPackage(pack, nodes.configs)) return [nodes.configs];
1420 		else return null;
1421 	}
1422 
1423 
1424 	protected override TreeNodes[] getChildren(TreeNode node)
1425 	{
1426 		if (auto pc = node in m_children)
1427 			return *pc;
1428 		auto ret = getChildrenRaw(node);
1429 		m_children[node] = ret;
1430 		return ret;
1431 	}
1432 
1433 	private final TreeNodes[] getChildrenRaw(TreeNode node)
1434 	{
1435 		import std.array : appender;
1436 		auto ret = appender!(TreeNodes[]);
1437 		auto pack = getPackage(node.pack, node.config);
1438 		if (!pack) {
1439 			// this can hapen when the package description contains syntax errors
1440 			logDebug("Invalid package in dependency tree: %s %s", node.pack, node.config);
1441 			return null;
1442 		}
1443 		auto basepack = pack.basePackage;
1444 
1445 		foreach (d; pack.getAllDependenciesRange()) {
1446 			auto dbasename = getBasePackageName(d.name);
1447 
1448 			// detect dependencies to the root package (or sub packages thereof)
1449 			if (dbasename == basepack.name) {
1450 				auto absdeppath = d.spec.mapToPath(pack.path).path;
1451 				absdeppath.endsWithSlash = true;
1452 				auto subpack = m_dub.m_packageManager.getSubPackage(basepack, getSubPackageName(d.name), true);
1453 				if (subpack) {
1454 					auto desireddeppath = d.name == dbasename ? basepack.path : subpack.path;
1455 					desireddeppath.endsWithSlash = true;
1456 					enforce(d.spec.path.empty || absdeppath == desireddeppath,
1457 						format("Dependency from %s to root package references wrong path: %s vs. %s",
1458 							node.pack, absdeppath.toNativeString(), desireddeppath.toNativeString()));
1459 				}
1460 				ret ~= TreeNodes(d.name, node.config);
1461 				continue;
1462 			}
1463 
1464 			DependencyType dt;
1465 			if (d.spec.optional) {
1466 				if (d.spec.default_) dt = DependencyType.optionalDefault;
1467 				else dt = DependencyType.optional;
1468 			} else dt = DependencyType.required;
1469 
1470 			Dependency dspec = d.spec.mapToPath(pack.path);
1471 
1472 			// if not upgrading, use the selected version
1473 			if (!(m_options & UpgradeOptions.upgrade) && m_selectedVersions && m_selectedVersions.hasSelectedVersion(dbasename))
1474 				dspec = m_selectedVersions.getSelectedVersion(dbasename);
1475 
1476 			// keep selected optional dependencies and avoid non-selected optional-default dependencies by default
1477 			if (m_selectedVersions && !m_selectedVersions.bare) {
1478 				if (dt == DependencyType.optionalDefault && !m_selectedVersions.hasSelectedVersion(dbasename))
1479 					dt = DependencyType.optional;
1480 				else if (dt == DependencyType.optional && m_selectedVersions.hasSelectedVersion(dbasename))
1481 					dt = DependencyType.optionalDefault;
1482 			}
1483 
1484 			ret ~= TreeNodes(d.name, dspec, dt);
1485 		}
1486 		return ret.data;
1487 	}
1488 
1489 	protected override bool matches(Dependency configs, Dependency config)
1490 	{
1491 		if (!configs.path.empty) return configs.path == config.path;
1492 		return configs.merge(config).valid;
1493 	}
1494 
1495 	private Package getPackage(string name, Dependency dep)
1496 	{
1497 		auto key = PackageDependency(name, dep);
1498 		if (auto pp = key in m_packages)
1499 			return *pp;
1500 		auto p = getPackageRaw(name, dep);
1501 		m_packages[key] = p;
1502 		return p;
1503 	}
1504 
1505 	private Package getPackageRaw(string name, Dependency dep)
1506 	{
1507 		auto basename = getBasePackageName(name);
1508 
1509 		// for sub packages, first try to get them from the base package
1510 		if (basename != name) {
1511 			auto subname = getSubPackageName(name);
1512 			auto basepack = getPackage(basename, dep);
1513 			if (!basepack) return null;
1514 			if (auto sp = m_dub.m_packageManager.getSubPackage(basepack, subname, true)) {
1515 				return sp;
1516 			} else if (!basepack.subPackages.canFind!(p => p.path.length)) {
1517 				// note: external sub packages are handled further below
1518 				auto spr = basepack.getInternalSubPackage(subname);
1519 				if (!spr.isNull) {
1520 					auto sp = new Package(spr, basepack.path, basepack);
1521 					m_remotePackages[sp.name] = sp;
1522 					return sp;
1523 				} else {
1524 					logDiagnostic("Sub package %s doesn't exist in %s %s.", name, basename, dep.version_);
1525 					return null;
1526 				}
1527 			} else if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep)) {
1528 				return ret;
1529 			} else {
1530 				logDiagnostic("External sub package %s %s not found.", name, dep.version_);
1531 				return null;
1532 			}
1533 		}
1534 
1535 		// shortcut if the referenced package is the root package
1536 		if (basename == m_rootPackage.basePackage.name)
1537 			return m_rootPackage.basePackage;
1538 
1539 		if (!dep.path.empty) {
1540 			try {
1541 				auto ret = m_dub.packageManager.getOrLoadPackage(dep.path);
1542 				if (dep.matches(ret.version_)) return ret;
1543 			} catch (Exception e) {
1544 				logDiagnostic("Failed to load path based dependency %s: %s", name, e.msg);
1545 				logDebug("Full error: %s", e.toString().sanitize);
1546 				return null;
1547 			}
1548 		}
1549 
1550 		if (auto ret = m_dub.m_packageManager.getBestPackage(name, dep))
1551 			return ret;
1552 
1553 		auto key = name ~ ":" ~ dep.version_.toString();
1554 		if (auto ret = key in m_remotePackages)
1555 			return *ret;
1556 
1557 		auto prerelease = (m_options & UpgradeOptions.preRelease) != 0;
1558 
1559 		auto rootpack = name.split(":")[0];
1560 
1561 		foreach (ps; m_dub.m_packageSuppliers) {
1562 			if (rootpack == name) {
1563 				try {
1564 					auto desc = ps.fetchPackageRecipe(name, dep, prerelease);
1565 					if (desc.type == Json.Type.null_)
1566 						continue;
1567 					auto ret = new Package(desc);
1568 					m_remotePackages[key] = ret;
1569 					return ret;
1570 				} catch (Exception e) {
1571 					logDiagnostic("Metadata for %s %s could not be downloaded from %s: %s", name, dep, ps.description, e.msg);
1572 					logDebug("Full error: %s", e.toString().sanitize);
1573 				}
1574 			} else {
1575 				logDiagnostic("Package %s not found in base package description (%s). Downloading whole package.", name, dep.version_.toString());
1576 				try {
1577 					FetchOptions fetchOpts;
1578 					fetchOpts |= prerelease ? FetchOptions.usePrerelease : FetchOptions.none;
1579 					m_dub.fetch(rootpack, dep, m_dub.defaultPlacementLocation, fetchOpts, "need sub package description");
1580 					auto ret = m_dub.m_packageManager.getBestPackage(name, dep);
1581 					if (!ret) {
1582 						logWarn("Package %s %s doesn't have a sub package %s", rootpack, dep.version_, name);
1583 						return null;
1584 					}
1585 					m_remotePackages[key] = ret;
1586 					return ret;
1587 				} catch (Exception e) {
1588 					logDiagnostic("Package %s could not be downloaded from %s: %s", rootpack, ps.description, e.msg);
1589 					logDebug("Full error: %s", e.toString().sanitize);
1590 				}
1591 			}
1592 		}
1593 
1594 		m_remotePackages[key] = null;
1595 
1596 		logWarn("Package %s %s could not be loaded either locally, or from the configured package registries.", name, dep);
1597 		return null;
1598 	}
1599 }
1600 
1601 private struct SpecialDirs {
1602 	NativePath temp;
1603 	NativePath userSettings;
1604 	NativePath systemSettings;
1605 	NativePath localRepository;
1606 }
1607 
1608 private class DubConfig {
1609 	private {
1610 		DubConfig m_parentConfig;
1611 		Json m_data;
1612 	}
1613 
1614 	this(Json data, DubConfig parent_config)
1615 	{
1616 		m_data = data;
1617 		m_parentConfig = parent_config;
1618 	}
1619 
1620 	@property string[] registryURLs()
1621 	{
1622 		string[] ret;
1623 		if (auto pv = "registryUrls" in m_data)
1624 			ret = (*pv).deserializeJson!(string[]);
1625 		if (m_parentConfig) ret ~= m_parentConfig.registryURLs;
1626 		return ret;
1627 	}
1628 
1629 	@property string defaultCompiler()
1630 	const {
1631 		if (auto pv = "defaultCompiler" in m_data)
1632 			return pv.get!string;
1633 		if (m_parentConfig) return m_parentConfig.defaultCompiler;
1634 		return null;
1635 	}
1636 
1637 	@property string defaultArchitecture()
1638 	const {
1639 		if(auto pv = "defaultArchitecture" in m_data)
1640 			return (*pv).get!string;
1641 		if (m_parentConfig) return m_parentConfig.defaultArchitecture;
1642 		return null;
1643 	}
1644 }