1 /**
2 	Management of packages on the local computer.
3 
4 	Copyright: © 2012-2016 rejectedsoftware e.K.
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig, Matthias Dondorff
7 */
8 module dub.packagemanager;
9 
10 import dub.dependency;
11 import dub.internal.utils;
12 import dub.internal.vibecompat.core.file;
13 import dub.internal.vibecompat.core.log;
14 import dub.internal.vibecompat.data.json;
15 import dub.internal.vibecompat.inet.path;
16 import dub.package_;
17 
18 import std.algorithm : countUntil, filter, sort, canFind, remove;
19 import std.array;
20 import std.conv;
21 import std.digest.sha;
22 import std.encoding : sanitize;
23 import std.exception;
24 import std.file;
25 import std.string;
26 import std.zip;
27 
28 
29 /// The PackageManager can retrieve present packages and get / remove
30 /// packages.
31 class PackageManager {
32 	private {
33 		Repository[] m_repositories;
34 		NativePath[] m_searchPath;
35 		Package[] m_packages;
36 		Package[] m_temporaryPackages;
37 		bool m_disableDefaultSearchPaths = false;
38 	}
39 
40 	/**
41 	   Instantiate an instance with a single search path
42 
43 	   This constructor is used when dub is invoked with the '--bar' CLI switch.
44 	   The instance will not look up the default repositories
45 	   (e.g. ~/.dub/packages), using only `path` instead.
46 
47 	   Params:
48 		 path = Path of the single repository
49 	 */
50 	this(NativePath path)
51 	{
52 		this.m_searchPath = [ path ];
53 		this.m_disableDefaultSearchPaths = true;
54 		this.refresh(true);
55 	}
56 
57 	deprecated("Use the overload which accepts 3 `NativePath` arguments")
58 	this(NativePath user_path, NativePath system_path, bool refresh_packages = true)
59 	{
60 		m_repositories = [
61 			Repository(user_path ~ "packages/"),
62 			Repository(system_path ~ "packages/")];
63 
64 		if (refresh_packages) refresh(true);
65 	}
66 
67 	this(NativePath package_path, NativePath user_path, NativePath system_path, bool refresh_packages = true)
68 	{
69 		m_repositories = [
70 			Repository(package_path ~ ".dub/packages/"),
71 			Repository(user_path ~ "packages/"),
72 			Repository(system_path ~ "packages/")];
73 
74 		if (refresh_packages) refresh(true);
75 	}
76 
77 	/** Gets/sets the list of paths to search for local packages.
78 	*/
79 	@property void searchPath(NativePath[] paths)
80 	{
81 		if (paths == m_searchPath) return;
82 		m_searchPath = paths.dup;
83 		refresh(false);
84 	}
85 	/// ditto
86 	@property const(NativePath)[] searchPath() const { return m_searchPath; }
87 
88 	/** Disables searching DUB's predefined search paths.
89 	*/
90 	deprecated("Instantiate a PackageManager instance with the single-argument constructor: `new PackageManager(path)`")
91 	@property void disableDefaultSearchPaths(bool val)
92 	{
93 		this._disableDefaultSearchPaths(val);
94 	}
95 
96 	// Non deprecated instance of the previous symbol,
97 	// as `Dub.updatePackageSearchPath` calls it and while nothing in Dub app
98 	// itself relies on it, just removing the call from `updatePackageSearchPath`
99 	// could break the library use case.
100 	package(dub) void _disableDefaultSearchPaths(bool val)
101 	{
102 		if (val == m_disableDefaultSearchPaths) return;
103 		m_disableDefaultSearchPaths = val;
104 		refresh(true);
105 	}
106 
107 	/** Returns the effective list of search paths, including default ones.
108 	*/
109 	@property const(NativePath)[] completeSearchPath()
110 	const {
111 		auto ret = appender!(NativePath[])();
112 		ret.put(cast(NativePath[])m_searchPath); // work around Phobos 17251
113 		if (!m_disableDefaultSearchPaths) {
114 			foreach (ref repo; m_repositories) {
115 				ret.put(cast(NativePath[])repo.searchPath);
116 				ret.put(cast(NativePath)repo.packagePath);
117 			}
118 		}
119 		return ret.data;
120 	}
121 
122 	/** Sets additional (read-only) package cache paths to search for packages.
123 
124 		Cache paths have the same structure as the default cache paths, such as
125 		".dub/packages/".
126 
127 		Note that previously set custom paths will be removed when setting this
128 		property.
129 	*/
130 	@property void customCachePaths(NativePath[] custom_cache_paths)
131 	{
132 		import std.algorithm.iteration : map;
133 		import std.array : array;
134 
135 		m_repositories.length = LocalPackageType.max+1;
136 		m_repositories ~= custom_cache_paths.map!(p => Repository(p)).array;
137 
138 		refresh(false);
139 	}
140 
141 
142 	/** Looks up a specific package.
143 
144 		Looks up a package matching the given version/path in the set of
145 		registered packages. The lookup order is done according the the
146 		usual rules (see getPackageIterator).
147 
148 		Params:
149 			name = The name of the package
150 			ver = The exact version of the package to query
151 			path = An exact path that the package must reside in. Note that
152 				the package must still be registered in the package manager.
153 			enable_overrides = Apply the local package override list before
154 				returning a package (enabled by default)
155 
156 		Returns:
157 			The matching package or null if no match was found.
158 	*/
159 	Package getPackage(string name, Version ver, bool enable_overrides = true)
160 	{
161 		if (enable_overrides) {
162 			foreach (ref repo; m_repositories)
163 				foreach (ovr; repo.overrides)
164 					if (ovr.package_ == name && ovr.version_.matches(ver)) {
165 						Package pack;
166 						if (!ovr.targetPath.empty) pack = getOrLoadPackage(ovr.targetPath);
167 						else pack = getPackage(name, ovr.targetVersion, false);
168 						if (pack) return pack;
169 
170 						logWarn("Package override %s %s -> %s %s doesn't reference an existing package.",
171 							ovr.package_, ovr.version_, ovr.targetVersion, ovr.targetPath);
172 					}
173 		}
174 
175 		foreach (p; getPackageIterator(name))
176 			if (p.version_ == ver)
177 				return p;
178 
179 		return null;
180 	}
181 
182 	/// ditto
183 	Package getPackage(string name, string ver, bool enable_overrides = true)
184 	{
185 		return getPackage(name, Version(ver), enable_overrides);
186 	}
187 
188 	/// ditto
189 	Package getPackage(string name, Version ver, NativePath path)
190 	{
191 		auto ret = getPackage(name, path);
192 		if (!ret || ret.version_ != ver) return null;
193 		return ret;
194 	}
195 
196 	/// ditto
197 	Package getPackage(string name, string ver, NativePath path)
198 	{
199 		return getPackage(name, Version(ver), path);
200 	}
201 
202 	/// ditto
203 	Package getPackage(string name, NativePath path)
204 	{
205 		foreach( p; getPackageIterator(name) )
206 			if (p.path.startsWith(path))
207 				return p;
208 		return null;
209 	}
210 
211 
212 	/** Looks up the first package matching the given name.
213 	*/
214 	Package getFirstPackage(string name)
215 	{
216 		foreach (ep; getPackageIterator(name))
217 			return ep;
218 		return null;
219 	}
220 
221 	/** For a given package path, returns the corresponding package.
222 
223 		If the package is already loaded, a reference is returned. Otherwise
224 		the package gets loaded and cached for the next call to this function.
225 
226 		Params:
227 			path = NativePath to the root directory of the package
228 			recipe_path = Optional path to the recipe file of the package
229 			allow_sub_packages = Also return a sub package if it resides in the given folder
230 
231 		Returns: The packages loaded from the given path
232 		Throws: Throws an exception if no package can be loaded
233 	*/
234 	Package getOrLoadPackage(NativePath path, NativePath recipe_path = NativePath.init, bool allow_sub_packages = false)
235 	{
236 		path.endsWithSlash = true;
237 		foreach (p; getPackageIterator())
238 			if (p.path == path && (!p.parentPackage || (allow_sub_packages && p.parentPackage.path != p.path)))
239 				return p;
240 		auto pack = Package.load(path, recipe_path);
241 		addPackages(m_temporaryPackages, pack);
242 		return pack;
243 	}
244 
245 
246 	/** Searches for the latest version of a package matching the given dependency.
247 	*/
248 	Package getBestPackage(string name, Dependency version_spec, bool enable_overrides = true)
249 	{
250 		Package ret;
251 		foreach (p; getPackageIterator(name))
252 			if (version_spec.matches(p.version_) && (!ret || p.version_ > ret.version_))
253 				ret = p;
254 
255 		if (enable_overrides && ret) {
256 			if (auto ovr = getPackage(name, ret.version_))
257 				return ovr;
258 		}
259 		return ret;
260 	}
261 
262 	/// ditto
263 	Package getBestPackage(string name, string version_spec)
264 	{
265 		return getBestPackage(name, Dependency(version_spec));
266 	}
267 
268 	/** Gets the a specific sub package.
269 
270 		In contrast to `Package.getSubPackage`, this function supports path
271 		based sub packages.
272 
273 		Params:
274 			base_package = The package from which to get a sub package
275 			sub_name = Name of the sub package (not prefixed with the base
276 				package name)
277 			silent_fail = If set to true, the function will return `null` if no
278 				package is found. Otherwise will throw an exception.
279 
280 	*/
281 	Package getSubPackage(Package base_package, string sub_name, bool silent_fail)
282 	{
283 		foreach (p; getPackageIterator(base_package.name~":"~sub_name))
284 			if (p.parentPackage is base_package)
285 				return p;
286 		enforce(silent_fail, "Sub package \""~base_package.name~":"~sub_name~"\" doesn't exist.");
287 		return null;
288 	}
289 
290 
291 	/** Determines if a package is managed by DUB.
292 
293 		Managed packages can be upgraded and removed.
294 	*/
295 	bool isManagedPackage(Package pack)
296 	const {
297 		auto ppath = pack.basePackage.path;
298 		return isManagedPath(ppath);
299 	}
300 
301 	/** Determines if a specific path is within a DUB managed package folder.
302 
303 		By default, managed folders are "~/.dub/packages" and
304 		"/var/lib/dub/packages".
305 	*/
306 	bool isManagedPath(NativePath path)
307 	const {
308 		foreach (rep; m_repositories) {
309 			NativePath rpath = rep.packagePath;
310 			if (path.startsWith(rpath))
311 				return true;
312 		}
313 		return false;
314 	}
315 
316 	/** Enables iteration over all known local packages.
317 
318 		Returns: A delegate suitable for use with `foreach` is returned.
319 	*/
320 	int delegate(int delegate(ref Package)) getPackageIterator()
321 	{
322 		int iterator(int delegate(ref Package) del)
323 		{
324 			foreach (tp; m_temporaryPackages)
325 				if (auto ret = del(tp)) return ret;
326 
327 			// first search local packages
328 			foreach (ref repo; m_repositories)
329 				foreach (p; repo.localPackages)
330 					if (auto ret = del(p)) return ret;
331 
332 			// and then all packages gathered from the search path
333 			foreach( p; m_packages )
334 				if( auto ret = del(p) )
335 					return ret;
336 			return 0;
337 		}
338 
339 		return &iterator;
340 	}
341 
342 	/** Enables iteration over all known local packages with a certain name.
343 
344 		Returns: A delegate suitable for use with `foreach` is returned.
345 	*/
346 	int delegate(int delegate(ref Package)) getPackageIterator(string name)
347 	{
348 		int iterator(int delegate(ref Package) del)
349 		{
350 			foreach (p; getPackageIterator())
351 				if (p.name == name)
352 					if (auto ret = del(p)) return ret;
353 			return 0;
354 		}
355 
356 		return &iterator;
357 	}
358 
359 
360 	/** Returns a list of all package overrides for the given scope.
361 	*/
362 	const(PackageOverride)[] getOverrides(LocalPackageType scope_)
363 	const {
364 		return m_repositories[scope_].overrides;
365 	}
366 
367 	/** Adds a new override for the given package.
368 	*/
369 	void addOverride(LocalPackageType scope_, string package_, Dependency version_spec, Version target)
370 	{
371 		m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target);
372 		writeLocalPackageOverridesFile(scope_);
373 	}
374 	/// ditto
375 	void addOverride(LocalPackageType scope_, string package_, Dependency version_spec, NativePath target)
376 	{
377 		m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target);
378 		writeLocalPackageOverridesFile(scope_);
379 	}
380 
381 	/** Removes an existing package override.
382 	*/
383 	void removeOverride(LocalPackageType scope_, string package_, Dependency version_spec)
384 	{
385 		Repository* rep = &m_repositories[scope_];
386 		foreach (i, ovr; rep.overrides) {
387 			if (ovr.package_ != package_ || ovr.version_ != version_spec)
388 				continue;
389 			rep.overrides = rep.overrides[0 .. i] ~ rep.overrides[i+1 .. $];
390 			writeLocalPackageOverridesFile(scope_);
391 			return;
392 		}
393 		throw new Exception(format("No override exists for %s %s", package_, version_spec));
394 	}
395 
396 	/// Extracts the package supplied as a path to it's zip file to the
397 	/// destination and sets a version field in the package description.
398 	Package storeFetchedPackage(NativePath zip_file_path, Json package_info, NativePath destination)
399 	{
400 		import std.range : walkLength;
401 
402 		auto package_name = package_info["name"].get!string;
403 		auto package_version = package_info["version"].get!string;
404 
405 		logDebug("Placing package '%s' version '%s' to location '%s' from file '%s'",
406 			package_name, package_version, destination.toNativeString(), zip_file_path.toNativeString());
407 
408 		if( existsFile(destination) ){
409 			throw new Exception(format("%s (%s) needs to be removed from '%s' prior placement.", package_name, package_version, destination));
410 		}
411 
412 		// open zip file
413 		ZipArchive archive;
414 		{
415 			logDebug("Opening file %s", zip_file_path);
416 			auto f = openFile(zip_file_path, FileMode.read);
417 			scope(exit) f.close();
418 			archive = new ZipArchive(f.readAll());
419 		}
420 
421 		logDebug("Extracting from zip.");
422 
423 		// In a github zip, the actual contents are in a subfolder
424 		alias PSegment = typeof(NativePath.init.head);
425 		PSegment[] zip_prefix;
426 		outer: foreach(ArchiveMember am; archive.directory) {
427 			auto path = NativePath(am.name).bySegment.array;
428 			foreach (fil; packageInfoFiles)
429 				if (path.length == 2 && path[$-1].name == fil.filename) {
430 					zip_prefix = path[0 .. $-1];
431 					break outer;
432 				}
433 		}
434 
435 		logDebug("zip root folder: %s", zip_prefix);
436 
437 		NativePath getCleanedPath(string fileName) {
438 			auto path = NativePath(fileName);
439 			if (zip_prefix.length && !path.bySegment.startsWith(zip_prefix)) return NativePath.init;
440 			static if (is(typeof(path[0 .. 1]))) return path[zip_prefix.length .. $];
441 			else return NativePath(path.bySegment.array[zip_prefix.length .. $]);
442 		}
443 
444 		static void setAttributes(string path, ArchiveMember am)
445 		{
446 			import std.datetime : DosFileTimeToSysTime;
447 
448 			auto mtime = DosFileTimeToSysTime(am.time);
449 			setTimes(path, mtime, mtime);
450 			if (auto attrs = am.fileAttributes)
451 				std.file.setAttributes(path, attrs);
452 		}
453 
454 		// extract & place
455 		mkdirRecurse(destination.toNativeString());
456 		logDebug("Copying all files...");
457 		int countFiles = 0;
458 		foreach(ArchiveMember a; archive.directory) {
459 			auto cleanedPath = getCleanedPath(a.name);
460 			if(cleanedPath.empty) continue;
461 			auto dst_path = destination ~ cleanedPath;
462 
463 			logDebug("Creating %s", cleanedPath);
464 			if( dst_path.endsWithSlash ){
465 				if( !existsDirectory(dst_path) )
466 					mkdirRecurse(dst_path.toNativeString());
467 			} else {
468 				if( !existsDirectory(dst_path.parentPath) )
469 					mkdirRecurse(dst_path.parentPath.toNativeString());
470 				{
471 					auto dstFile = openFile(dst_path, FileMode.createTrunc);
472 					scope(exit) dstFile.close();
473 					dstFile.put(archive.expand(a));
474 				}
475 				setAttributes(dst_path.toNativeString(), a);
476 				++countFiles;
477 			}
478 		}
479 		logDebug("%s file(s) copied.", to!string(countFiles));
480 
481 		// overwrite dub.json (this one includes a version field)
482 		auto pack = Package.load(destination, NativePath.init, null, package_info["version"].get!string);
483 
484 		if (pack.recipePath.head != defaultPackageFilename)
485 			// Storeinfo saved a default file, this could be different to the file from the zip.
486 			removeFile(pack.recipePath);
487 		pack.storeInfo();
488 		addPackages(m_packages, pack);
489 		return pack;
490 	}
491 
492 	/// Removes the given the package.
493 	void remove(in Package pack)
494 	{
495 		logDebug("Remove %s, version %s, path '%s'", pack.name, pack.version_, pack.path);
496 		enforce(!pack.path.empty, "Cannot remove package "~pack.name~" without a path.");
497 
498 		// remove package from repositories' list
499 		bool found = false;
500 		bool removeFrom(Package[] packs, in Package pack) {
501 			auto packPos = countUntil!("a.path == b.path")(packs, pack);
502 			if(packPos != -1) {
503 				packs = .remove(packs, packPos);
504 				return true;
505 			}
506 			return false;
507 		}
508 		foreach(repo; m_repositories) {
509 			if(removeFrom(repo.localPackages, pack)) {
510 				found = true;
511 				break;
512 			}
513 		}
514 		if(!found)
515 			found = removeFrom(m_packages, pack);
516 		enforce(found, "Cannot remove, package not found: '"~ pack.name ~"', path: " ~ to!string(pack.path));
517 
518 		logDebug("About to delete root folder for package '%s'.", pack.path);
519 		rmdirRecurse(pack.path.toNativeString());
520 		logInfo("Removed package: '"~pack.name~"'");
521 	}
522 
523 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
524 	deprecated("Use `remove(pack)` directly instead, the boolean has no effect")
525 	void remove(in Package pack, bool force_remove)
526 	{
527 		remove(pack);
528 	}
529 
530 	Package addLocalPackage(NativePath path, string verName, LocalPackageType type)
531 	{
532 		path.endsWithSlash = true;
533 		auto pack = Package.load(path);
534 		enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString());
535 		if (verName.length)
536 			pack.version_ = Version(verName);
537 
538 		// don't double-add packages
539 		Package[]* packs = &m_repositories[type].localPackages;
540 		foreach (p; *packs) {
541 			if (p.path == path) {
542 				enforce(p.version_ == pack.version_, "Adding the same local package twice with differing versions is not allowed.");
543 				logInfo("Package is already registered: %s (version: %s)", p.name, p.version_);
544 				return p;
545 			}
546 		}
547 
548 		addPackages(*packs, pack);
549 
550 		writeLocalPackageList(type);
551 
552 		logInfo("Registered package: %s (version: %s)", pack.name, pack.version_);
553 		return pack;
554 	}
555 
556 	void removeLocalPackage(NativePath path, LocalPackageType type)
557 	{
558 		path.endsWithSlash = true;
559 
560 		Package[]* packs = &m_repositories[type].localPackages;
561 		size_t[] to_remove;
562 		foreach( i, entry; *packs )
563 			if( entry.path == path )
564 				to_remove ~= i;
565 		enforce(to_remove.length > 0, "No "~type.to!string()~" package found at "~path.toNativeString());
566 
567 		string[Version] removed;
568 		foreach_reverse( i; to_remove ) {
569 			removed[(*packs)[i].version_] = (*packs)[i].name;
570 			*packs = (*packs)[0 .. i] ~ (*packs)[i+1 .. $];
571 		}
572 
573 		writeLocalPackageList(type);
574 
575 		foreach(ver, name; removed)
576 			logInfo("Deregistered package: %s (version: %s)", name, ver);
577 	}
578 
579 	/// For the given type add another path where packages will be looked up.
580 	void addSearchPath(NativePath path, LocalPackageType type)
581 	{
582 		m_repositories[type].searchPath ~= path;
583 		writeLocalPackageList(type);
584 	}
585 
586 	/// Removes a search path from the given type.
587 	void removeSearchPath(NativePath path, LocalPackageType type)
588 	{
589 		m_repositories[type].searchPath = m_repositories[type].searchPath.filter!(p => p != path)().array();
590 		writeLocalPackageList(type);
591 	}
592 
593 	void refresh(bool refresh_existing_packages)
594 	{
595 		logDiagnostic("Refreshing local packages (refresh existing: %s)...", refresh_existing_packages);
596 
597 		// load locally defined packages
598 		void scanLocalPackages(LocalPackageType type)
599 		{
600 			NativePath list_path = m_repositories[type].packagePath;
601 			Package[] packs;
602 			NativePath[] paths;
603 			try {
604 				auto local_package_file = list_path ~ LocalPackagesFilename;
605 				logDiagnostic("Looking for local package map at %s", local_package_file.toNativeString());
606 				if( !existsFile(local_package_file) ) return;
607 				logDiagnostic("Try to load local package map at %s", local_package_file.toNativeString());
608 				auto packlist = jsonFromFile(list_path ~ LocalPackagesFilename);
609 				enforce(packlist.type == Json.Type.array, LocalPackagesFilename~" must contain an array.");
610 				foreach( pentry; packlist ){
611 					try {
612 						auto name = pentry["name"].get!string;
613 						auto path = NativePath(pentry["path"].get!string);
614 						if (name == "*") {
615 							paths ~= path;
616 						} else {
617 							auto ver = Version(pentry["version"].get!string);
618 
619 							Package pp;
620 							if (!refresh_existing_packages) {
621 								foreach (p; m_repositories[type].localPackages)
622 									if (p.path == path) {
623 										pp = p;
624 										break;
625 									}
626 							}
627 
628 							if (!pp) {
629 								auto infoFile = Package.findPackageFile(path);
630 								if (!infoFile.empty) pp = Package.load(path, infoFile);
631 								else {
632 									logWarn("Locally registered package %s %s was not found. Please run 'dub remove-local \"%s\"'.",
633 										name, ver, path.toNativeString());
634 									auto info = Json.emptyObject;
635 									info["name"] = name;
636 									pp = new Package(info, path);
637 								}
638 							}
639 
640 							if (pp.name != name)
641 								logWarn("Local package at %s has different name than %s (%s)", path.toNativeString(), name, pp.name);
642 							pp.version_ = ver;
643 
644 							addPackages(packs, pp);
645 						}
646 					} catch( Exception e ){
647 						logWarn("Error adding local package: %s", e.msg);
648 					}
649 				}
650 			} catch( Exception e ){
651 				logDiagnostic("Loading of local package list at %s failed: %s", list_path.toNativeString(), e.msg);
652 			}
653 			m_repositories[type].localPackages = packs;
654 			m_repositories[type].searchPath = paths;
655 		}
656 		if (!m_disableDefaultSearchPaths)
657 		{
658 			scanLocalPackages(LocalPackageType.system);
659 			scanLocalPackages(LocalPackageType.user);
660 			scanLocalPackages(LocalPackageType.package_);
661 		}
662 
663 		auto old_packages = m_packages;
664 
665 		// rescan the system and user package folder
666 		void scanPackageFolder(NativePath path)
667 		{
668 			if( path.existsDirectory() ){
669 				logDebug("iterating dir %s", path.toNativeString());
670 				try foreach( pdir; iterateDirectory(path) ){
671 					logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
672 					if (!pdir.isDirectory) continue;
673 
674 					auto pack_path = path ~ (pdir.name ~ "/");
675 
676 					auto packageFile = Package.findPackageFile(pack_path);
677 
678 					if (isManagedPath(path) && packageFile.empty) {
679 						// Search for a single directory within this directory which happen to be a prefix of pdir
680 						// This is to support new folder structure installed over the ancient one.
681 						foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/")))
682 							if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) {// eg: package vibe-d will be in "vibe-d-x.y.z/vibe-d"
683 								pack_path ~= subdir.name ~ "/";
684 								packageFile = Package.findPackageFile(pack_path);
685 								break;
686 							}
687 					}
688 
689 					if (packageFile.empty) continue;
690 					Package p;
691 					try {
692 						if (!refresh_existing_packages)
693 							foreach (pp; old_packages)
694 								if (pp.path == pack_path) {
695 									p = pp;
696 									break;
697 								}
698 						if (!p) p = Package.load(pack_path, packageFile);
699 						addPackages(m_packages, p);
700 					} catch( Exception e ){
701 						logError("Failed to load package in %s: %s", pack_path, e.msg);
702 						logDiagnostic("Full error: %s", e.toString().sanitize());
703 					}
704 				}
705 				catch(Exception e) logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString());
706 			}
707 		}
708 
709 		m_packages = null;
710 		foreach (p; this.completeSearchPath)
711 			scanPackageFolder(p);
712 
713 		void loadOverrides(LocalPackageType type)
714 		{
715 			m_repositories[type].overrides = null;
716 			auto ovrfilepath = m_repositories[type].packagePath ~ LocalOverridesFilename;
717 			if (existsFile(ovrfilepath)) {
718 				foreach (entry; jsonFromFile(ovrfilepath)) {
719 					PackageOverride ovr;
720 					ovr.package_ = entry["name"].get!string;
721 					ovr.version_ = Dependency(entry["version"].get!string);
722 					if (auto pv = "targetVersion" in entry) ovr.targetVersion = Version(pv.get!string);
723 					if (auto pv = "targetPath" in entry) ovr.targetPath = NativePath(pv.get!string);
724 					m_repositories[type].overrides ~= ovr;
725 				}
726 			}
727 		}
728 		if (!m_disableDefaultSearchPaths)
729 		{
730 			loadOverrides(LocalPackageType.package_);
731 			loadOverrides(LocalPackageType.user);
732 			loadOverrides(LocalPackageType.system);
733 		}
734 	}
735 
736 	alias Hash = ubyte[];
737 	/// Generates a hash value for a given package.
738 	/// Some files or folders are ignored during the generation (like .dub and
739 	/// .svn folders)
740 	Hash hashPackage(Package pack)
741 	{
742 		string[] ignored_directories = [".git", ".dub", ".svn"];
743 		// something from .dub_ignore or what?
744 		string[] ignored_files = [];
745 		SHA1 sha1;
746 		foreach(file; dirEntries(pack.path.toNativeString(), SpanMode.depth)) {
747 			if(file.isDir && ignored_directories.canFind(NativePath(file.name).head.name))
748 				continue;
749 			else if(ignored_files.canFind(NativePath(file.name).head.name))
750 				continue;
751 
752 			sha1.put(cast(ubyte[])NativePath(file.name).head.name);
753 			if(file.isDir) {
754 				logDebug("Hashed directory name %s", NativePath(file.name).head);
755 			}
756 			else {
757 				sha1.put(openFile(NativePath(file.name)).readAll());
758 				logDebug("Hashed file contents from %s", NativePath(file.name).head);
759 			}
760 		}
761 		auto hash = sha1.finish();
762 		logDebug("Project hash: %s", hash);
763 		return hash[].dup;
764 	}
765 
766 	private void writeLocalPackageList(LocalPackageType type)
767 	{
768 		Json[] newlist;
769 		foreach (p; m_repositories[type].searchPath) {
770 			auto entry = Json.emptyObject;
771 			entry["name"] = "*";
772 			entry["path"] = p.toNativeString();
773 			newlist ~= entry;
774 		}
775 
776 		foreach (p; m_repositories[type].localPackages) {
777 			if (p.parentPackage) continue; // do not store sub packages
778 			auto entry = Json.emptyObject;
779 			entry["name"] = p.name;
780 			entry["version"] = p.version_.toString();
781 			entry["path"] = p.path.toNativeString();
782 			newlist ~= entry;
783 		}
784 
785 		NativePath path = m_repositories[type].packagePath;
786 		if( !existsDirectory(path) ) mkdirRecurse(path.toNativeString());
787 		writeJsonFile(path ~ LocalPackagesFilename, Json(newlist));
788 	}
789 
790 	private void writeLocalPackageOverridesFile(LocalPackageType type)
791 	{
792 		Json[] newlist;
793 		foreach (ovr; m_repositories[type].overrides) {
794 			auto jovr = Json.emptyObject;
795 			jovr["name"] = ovr.package_;
796 			jovr["version"] = ovr.version_.versionSpec;
797 			if (!ovr.targetPath.empty) jovr["targetPath"] = ovr.targetPath.toNativeString();
798 			else jovr["targetVersion"] = ovr.targetVersion.toString();
799 			newlist ~= jovr;
800 		}
801 		auto path = m_repositories[type].packagePath;
802 		if (!existsDirectory(path)) mkdirRecurse(path.toNativeString());
803 		writeJsonFile(path ~ LocalOverridesFilename, Json(newlist));
804 	}
805 
806 	/// Adds the package and scans for subpackages.
807 	private void addPackages(ref Package[] dst_repos, Package pack)
808 	const {
809 		// Add the main package.
810 		dst_repos ~= pack;
811 
812 		// Additionally to the internally defined subpackages, whose metadata
813 		// is loaded with the main dub.json, load all externally defined
814 		// packages after the package is available with all the data.
815 		foreach (spr; pack.subPackages) {
816 			Package sp;
817 
818 			if (spr.path.length) {
819 				auto p = NativePath(spr.path);
820 				p.normalize();
821 				enforce(!p.absolute, "Sub package paths must be sub paths of the parent package.");
822 				auto path = pack.path ~ p;
823 				if (!existsFile(path)) {
824 					logError("Package %s declared a sub-package, definition file is missing: %s", pack.name, path.toNativeString());
825 					continue;
826 				}
827 				sp = Package.load(path, NativePath.init, pack);
828 			} else sp = new Package(spr.recipe, pack.path, pack);
829 
830 			// Add the subpackage.
831 			try {
832 				dst_repos ~= sp;
833 			} catch (Exception e) {
834 				logError("Package '%s': Failed to load sub-package %s: %s", pack.name,
835 					spr.path.length ? spr.path : spr.recipe.name, e.msg);
836 				logDiagnostic("Full error: %s", e.toString().sanitize());
837 			}
838 		}
839 	}
840 }
841 
842 struct PackageOverride {
843 	string package_;
844 	Dependency version_;
845 	Version targetVersion;
846 	NativePath targetPath;
847 
848 	this(string package_, Dependency version_, Version target_version)
849 	{
850 		this.package_ = package_;
851 		this.version_ = version_;
852 		this.targetVersion = target_version;
853 	}
854 
855 	this(string package_, Dependency version_, NativePath target_path)
856 	{
857 		this.package_ = package_;
858 		this.version_ = version_;
859 		this.targetPath = target_path;
860 	}
861 }
862 
863 enum LocalPackageType {
864 	package_,
865 	user,
866 	system
867 }
868 
869 private enum LocalPackagesFilename = "local-packages.json";
870 private enum LocalOverridesFilename = "local-overrides.json";
871 
872 
873 private struct Repository {
874 	NativePath packagePath;
875 	NativePath[] searchPath;
876 	Package[] localPackages;
877 	PackageOverride[] overrides;
878 
879 	this(NativePath path)
880 	{
881 		this.packagePath = path;
882 	}
883 }