1 /**
2 	Management of packages on the local computer.
3 
4 	Copyright: © 2012-2016 rejectedsoftware e.K.
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig, Matthias Dondorff
7 */
8 module dub.packagemanager;
9 
10 import dub.dependency;
11 import dub.internal.utils;
12 import dub.internal.vibecompat.core.file;
13 import dub.internal.vibecompat.core.log;
14 import dub.internal.vibecompat.data.json;
15 import dub.internal.vibecompat.inet.path;
16 import dub.package_;
17 
18 import std.algorithm : countUntil, filter, sort, canFind, remove;
19 import std.array;
20 import std.conv;
21 import std.digest.sha;
22 import std.encoding : sanitize;
23 import std.exception;
24 import std.file;
25 import std.string;
26 import std.zip;
27 
28 
29 /// The PackageManager can retrieve present packages and get / remove
30 /// packages.
31 class PackageManager {
32 	private {
33 		Repository[LocalPackageType] m_repositories;
34 		NativePath[] m_searchPath;
35 		Package[] m_packages;
36 		Package[] m_temporaryPackages;
37 		bool m_disableDefaultSearchPaths = false;
38 	}
39 
40 	this(NativePath user_path, NativePath system_path, bool refresh_packages = true)
41 	{
42 		m_repositories[LocalPackageType.user] = Repository(user_path);
43 		m_repositories[LocalPackageType.system] = Repository(system_path);
44 		if (refresh_packages) refresh(true);
45 	}
46 
47 	/** Gets/sets the list of paths to search for local packages.
48 	*/
49 	@property void searchPath(NativePath[] paths)
50 	{
51 		if (paths == m_searchPath) return;
52 		m_searchPath = paths.dup;
53 		refresh(false);
54 	}
55 	/// ditto
56 	@property const(NativePath)[] searchPath() const { return m_searchPath; }
57 
58 	/** Disables searching DUB's predefined search paths.
59 	*/
60 	@property void disableDefaultSearchPaths(bool val)
61 	{
62 		if (val == m_disableDefaultSearchPaths) return;
63 		m_disableDefaultSearchPaths = val;
64 		refresh(true);
65 	}
66 
67 	/** Returns the effective list of search paths, including default ones.
68 	*/
69 	@property const(NativePath)[] completeSearchPath()
70 	const {
71 		auto ret = appender!(NativePath[])();
72 		ret.put(cast(NativePath[])m_searchPath); // work around Phobos 17251
73 		if (!m_disableDefaultSearchPaths) {
74 			ret.put(cast(NativePath[])m_repositories[LocalPackageType.user].searchPath);
75 			ret.put(cast(NativePath)m_repositories[LocalPackageType.user].packagePath);
76 			ret.put(cast(NativePath[])m_repositories[LocalPackageType.system].searchPath);
77 			ret.put(cast(NativePath)m_repositories[LocalPackageType.system].packagePath);
78 		}
79 		return ret.data;
80 	}
81 
82 
83 	/** Looks up a specific package.
84 
85 		Looks up a package matching the given version/path in the set of
86 		registered packages. The lookup order is done according the the
87 		usual rules (see getPackageIterator).
88 
89 		Params:
90 			name = The name of the package
91 			ver = The exact version of the package to query
92 			path = An exact path that the package must reside in. Note that
93 				the package must still be registered in the package manager.
94 			enable_overrides = Apply the local package override list before
95 				returning a package (enabled by default)
96 
97 		Returns:
98 			The matching package or null if no match was found.
99 	*/
100 	Package getPackage(string name, Version ver, bool enable_overrides = true)
101 	{
102 		if (enable_overrides) {
103 			foreach (tp; [LocalPackageType.user, LocalPackageType.system])
104 				foreach (ovr; m_repositories[tp].overrides)
105 					if (ovr.package_ == name && ovr.version_.matches(ver)) {
106 						Package pack;
107 						if (!ovr.targetPath.empty) pack = getOrLoadPackage(ovr.targetPath);
108 						else pack = getPackage(name, ovr.targetVersion, false);
109 						if (pack) return pack;
110 
111 						logWarn("Package override %s %s -> %s %s doesn't reference an existing package.",
112 							ovr.package_, ovr.version_, ovr.targetVersion, ovr.targetPath);
113 					}
114 		}
115 
116 		foreach (p; getPackageIterator(name))
117 			if (p.version_ == ver)
118 				return p;
119 
120 		return null;
121 	}
122 
123 	/// ditto
124 	Package getPackage(string name, string ver, bool enable_overrides = true)
125 	{
126 		return getPackage(name, Version(ver), enable_overrides);
127 	}
128 
129 	/// ditto
130 	Package getPackage(string name, Version ver, NativePath path)
131 	{
132 		auto ret = getPackage(name, path);
133 		if (!ret || ret.version_ != ver) return null;
134 		return ret;
135 	}
136 
137 	/// ditto
138 	Package getPackage(string name, string ver, NativePath path)
139 	{
140 		return getPackage(name, Version(ver), path);
141 	}
142 
143 	/// ditto
144 	Package getPackage(string name, NativePath path)
145 	{
146 		foreach( p; getPackageIterator(name) )
147 			if (p.path.startsWith(path))
148 				return p;
149 		return null;
150 	}
151 
152 
153 	/** Looks up the first package matching the given name.
154 	*/
155 	Package getFirstPackage(string name)
156 	{
157 		foreach (ep; getPackageIterator(name))
158 			return ep;
159 		return null;
160 	}
161 
162 	/** For a given package path, returns the corresponding package.
163 
164 		If the package is already loaded, a reference is returned. Otherwise
165 		the package gets loaded and cached for the next call to this function.
166 
167 		Params:
168 			path = NativePath to the root directory of the package
169 			recipe_path = Optional path to the recipe file of the package
170 			allow_sub_packages = Also return a sub package if it resides in the given folder
171 
172 		Returns: The packages loaded from the given path
173 		Throws: Throws an exception if no package can be loaded
174 	*/
175 	Package getOrLoadPackage(NativePath path, NativePath recipe_path = NativePath.init, bool allow_sub_packages = false)
176 	{
177 		path.endsWithSlash = true;
178 		foreach (p; getPackageIterator())
179 			if (p.path == path && (!p.parentPackage || (allow_sub_packages && p.parentPackage.path != p.path)))
180 				return p;
181 		auto pack = Package.load(path, recipe_path);
182 		addPackages(m_temporaryPackages, pack);
183 		return pack;
184 	}
185 
186 
187 	/** Searches for the latest version of a package matching the given dependency.
188 	*/
189 	Package getBestPackage(string name, Dependency version_spec, bool enable_overrides = true)
190 	{
191 		Package ret;
192 		foreach (p; getPackageIterator(name))
193 			if (version_spec.matches(p.version_) && (!ret || p.version_ > ret.version_))
194 				ret = p;
195 
196 		if (enable_overrides && ret) {
197 			if (auto ovr = getPackage(name, ret.version_))
198 				return ovr;
199 		}
200 		return ret;
201 	}
202 
203 	/// ditto
204 	Package getBestPackage(string name, string version_spec)
205 	{
206 		return getBestPackage(name, Dependency(version_spec));
207 	}
208 
209 	/** Gets the a specific sub package.
210 
211 		In contrast to `Package.getSubPackage`, this function supports path
212 		based sub packages.
213 
214 		Params:
215 			base_package = The package from which to get a sub package
216 			sub_name = Name of the sub package (not prefixed with the base
217 				package name)
218 			silent_fail = If set to true, the function will return `null` if no
219 				package is found. Otherwise will throw an exception.
220 
221 	*/
222 	Package getSubPackage(Package base_package, string sub_name, bool silent_fail)
223 	{
224 		foreach (p; getPackageIterator(base_package.name~":"~sub_name))
225 			if (p.parentPackage is base_package)
226 				return p;
227 		enforce(silent_fail, "Sub package \""~base_package.name~":"~sub_name~"\" doesn't exist.");
228 		return null;
229 	}
230 
231 
232 	/** Determines if a package is managed by DUB.
233 
234 		Managed packages can be upgraded and removed.
235 	*/
236 	bool isManagedPackage(Package pack)
237 	const {
238 		auto ppath = pack.basePackage.path;
239 		return isManagedPath(ppath);
240 	}
241 
242 	/** Determines if a specific path is within a DUB managed package folder.
243 
244 		By default, managed folders are "~/.dub/packages" and
245 		"/var/lib/dub/packages".
246 	*/
247 	bool isManagedPath(NativePath path)
248 	const {
249 		foreach (rep; m_repositories) {
250 			NativePath rpath = rep.packagePath;
251 			if (path.startsWith(rpath))
252 				return true;
253 		}
254 		return false;
255 	}
256 
257 	/** Enables iteration over all known local packages.
258 
259 		Returns: A delegate suitable for use with `foreach` is returned.
260 	*/
261 	int delegate(int delegate(ref Package)) getPackageIterator()
262 	{
263 		int iterator(int delegate(ref Package) del)
264 		{
265 			foreach (tp; m_temporaryPackages)
266 				if (auto ret = del(tp)) return ret;
267 
268 			// first search local packages
269 			foreach (tp; LocalPackageType.min .. LocalPackageType.max+1)
270 				foreach (p; m_repositories[cast(LocalPackageType)tp].localPackages)
271 					if (auto ret = del(p)) return ret;
272 
273 			// and then all packages gathered from the search path
274 			foreach( p; m_packages )
275 				if( auto ret = del(p) )
276 					return ret;
277 			return 0;
278 		}
279 
280 		return &iterator;
281 	}
282 
283 	/** Enables iteration over all known local packages with a certain name.
284 
285 		Returns: A delegate suitable for use with `foreach` is returned.
286 	*/
287 	int delegate(int delegate(ref Package)) getPackageIterator(string name)
288 	{
289 		int iterator(int delegate(ref Package) del)
290 		{
291 			foreach (p; getPackageIterator())
292 				if (p.name == name)
293 					if (auto ret = del(p)) return ret;
294 			return 0;
295 		}
296 
297 		return &iterator;
298 	}
299 
300 
301 	/** Returns a list of all package overrides for the given scope.
302 	*/
303 	const(PackageOverride)[] getOverrides(LocalPackageType scope_)
304 	const {
305 		return m_repositories[scope_].overrides;
306 	}
307 
308 	/** Adds a new override for the given package.
309 	*/
310 	void addOverride(LocalPackageType scope_, string package_, Dependency version_spec, Version target)
311 	{
312 		m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target);
313 		writeLocalPackageOverridesFile(scope_);
314 	}
315 	/// ditto
316 	void addOverride(LocalPackageType scope_, string package_, Dependency version_spec, NativePath target)
317 	{
318 		m_repositories[scope_].overrides ~= PackageOverride(package_, version_spec, target);
319 		writeLocalPackageOverridesFile(scope_);
320 	}
321 
322 	/** Removes an existing package override.
323 	*/
324 	void removeOverride(LocalPackageType scope_, string package_, Dependency version_spec)
325 	{
326 		Repository* rep = &m_repositories[scope_];
327 		foreach (i, ovr; rep.overrides) {
328 			if (ovr.package_ != package_ || ovr.version_ != version_spec)
329 				continue;
330 			rep.overrides = rep.overrides[0 .. i] ~ rep.overrides[i+1 .. $];
331 			writeLocalPackageOverridesFile(scope_);
332 			return;
333 		}
334 		throw new Exception(format("No override exists for %s %s", package_, version_spec));
335 	}
336 
337 	/// Extracts the package supplied as a path to it's zip file to the
338 	/// destination and sets a version field in the package description.
339 	Package storeFetchedPackage(NativePath zip_file_path, Json package_info, NativePath destination)
340 	{
341 		import std.range : walkLength;
342 
343 		auto package_name = package_info["name"].get!string;
344 		auto package_version = package_info["version"].get!string;
345 		auto clean_package_version = package_version[package_version.startsWith("~") ? 1 : 0 .. $];
346 
347 		logDebug("Placing package '%s' version '%s' to location '%s' from file '%s'",
348 			package_name, package_version, destination.toNativeString(), zip_file_path.toNativeString());
349 
350 		if( existsFile(destination) ){
351 			throw new Exception(format("%s (%s) needs to be removed from '%s' prior placement.", package_name, package_version, destination));
352 		}
353 
354 		// open zip file
355 		ZipArchive archive;
356 		{
357 			logDebug("Opening file %s", zip_file_path);
358 			auto f = openFile(zip_file_path, FileMode.read);
359 			scope(exit) f.close();
360 			archive = new ZipArchive(f.readAll());
361 		}
362 
363 		logDebug("Extracting from zip.");
364 
365 		// In a github zip, the actual contents are in a subfolder
366 		alias PSegment = typeof(NativePath.init.head);
367 		PSegment[] zip_prefix;
368 		outer: foreach(ArchiveMember am; archive.directory) {
369 			auto path = NativePath(am.name).bySegment.array;
370 			foreach (fil; packageInfoFiles)
371 				if (path.length == 2 && path[$-1].toString == fil.filename) {
372 					zip_prefix = path[0 .. $-1];
373 					break outer;
374 				}
375 		}
376 
377 		logDebug("zip root folder: %s", zip_prefix);
378 
379 		NativePath getCleanedPath(string fileName) {
380 			auto path = NativePath(fileName);
381 			if (zip_prefix.length && !path.bySegment.startsWith(zip_prefix)) return NativePath.init;
382 			static if (is(typeof(path[0 .. 1]))) return path[zip_prefix.length .. $];
383 			else return NativePath(path.bySegment.array[zip_prefix.length .. $]);
384 		}
385 
386 		static void setAttributes(string path, ArchiveMember am)
387 		{
388 			import std.datetime : DosFileTimeToSysTime;
389 
390 			auto mtime = DosFileTimeToSysTime(am.time);
391 			setTimes(path, mtime, mtime);
392 			if (auto attrs = am.fileAttributes)
393 				std.file.setAttributes(path, attrs);
394 		}
395 
396 		// extract & place
397 		mkdirRecurse(destination.toNativeString());
398 		logDebug("Copying all files...");
399 		int countFiles = 0;
400 		foreach(ArchiveMember a; archive.directory) {
401 			auto cleanedPath = getCleanedPath(a.name);
402 			if(cleanedPath.empty) continue;
403 			auto dst_path = destination ~ cleanedPath;
404 
405 			logDebug("Creating %s", cleanedPath);
406 			if( dst_path.endsWithSlash ){
407 				if( !existsDirectory(dst_path) )
408 					mkdirRecurse(dst_path.toNativeString());
409 			} else {
410 				if( !existsDirectory(dst_path.parentPath) )
411 					mkdirRecurse(dst_path.parentPath.toNativeString());
412 				{
413 					auto dstFile = openFile(dst_path, FileMode.createTrunc);
414 					scope(exit) dstFile.close();
415 					dstFile.put(archive.expand(a));
416 				}
417 				setAttributes(dst_path.toNativeString(), a);
418 				++countFiles;
419 			}
420 		}
421 		logDebug("%s file(s) copied.", to!string(countFiles));
422 
423 		// overwrite dub.json (this one includes a version field)
424 		auto pack = Package.load(destination, NativePath.init, null, package_info["version"].get!string);
425 
426 		if (pack.recipePath.head != defaultPackageFilename)
427 			// Storeinfo saved a default file, this could be different to the file from the zip.
428 			removeFile(pack.recipePath);
429 		pack.storeInfo();
430 		addPackages(m_packages, pack);
431 		return pack;
432 	}
433 
434 	/// Removes the given the package.
435 	void remove(in Package pack)
436 	{
437 		logDebug("Remove %s, version %s, path '%s'", pack.name, pack.version_, pack.path);
438 		enforce(!pack.path.empty, "Cannot remove package "~pack.name~" without a path.");
439 
440 		// remove package from repositories' list
441 		bool found = false;
442 		bool removeFrom(Package[] packs, in Package pack) {
443 			auto packPos = countUntil!("a.path == b.path")(packs, pack);
444 			if(packPos != -1) {
445 				packs = .remove(packs, packPos);
446 				return true;
447 			}
448 			return false;
449 		}
450 		foreach(repo; m_repositories) {
451 			if(removeFrom(repo.localPackages, pack)) {
452 				found = true;
453 				break;
454 			}
455 		}
456 		if(!found)
457 			found = removeFrom(m_packages, pack);
458 		enforce(found, "Cannot remove, package not found: '"~ pack.name ~"', path: " ~ to!string(pack.path));
459 
460 		logDebug("About to delete root folder for package '%s'.", pack.path);
461 		rmdirRecurse(pack.path.toNativeString());
462 		logInfo("Removed package: '"~pack.name~"'");
463 	}
464 
465 	/// Compatibility overload. Use the version without a `force_remove` argument instead.
466 	void remove(in Package pack, bool force_remove)
467 	{
468 		remove(pack);
469 	}
470 
471 	Package addLocalPackage(NativePath path, string verName, LocalPackageType type)
472 	{
473 		path.endsWithSlash = true;
474 		auto pack = Package.load(path);
475 		enforce(pack.name.length, "The package has no name, defined in: " ~ path.toString());
476 		if (verName.length)
477 			pack.version_ = Version(verName);
478 
479 		// don't double-add packages
480 		Package[]* packs = &m_repositories[type].localPackages;
481 		foreach (p; *packs) {
482 			if (p.path == path) {
483 				enforce(p.version_ == pack.version_, "Adding the same local package twice with differing versions is not allowed.");
484 				logInfo("Package is already registered: %s (version: %s)", p.name, p.version_);
485 				return p;
486 			}
487 		}
488 
489 		addPackages(*packs, pack);
490 
491 		writeLocalPackageList(type);
492 
493 		logInfo("Registered package: %s (version: %s)", pack.name, pack.version_);
494 		return pack;
495 	}
496 
497 	void removeLocalPackage(NativePath path, LocalPackageType type)
498 	{
499 		path.endsWithSlash = true;
500 
501 		Package[]* packs = &m_repositories[type].localPackages;
502 		size_t[] to_remove;
503 		foreach( i, entry; *packs )
504 			if( entry.path == path )
505 				to_remove ~= i;
506 		enforce(to_remove.length > 0, "No "~type.to!string()~" package found at "~path.toNativeString());
507 
508 		string[Version] removed;
509 		foreach_reverse( i; to_remove ) {
510 			removed[(*packs)[i].version_] = (*packs)[i].name;
511 			*packs = (*packs)[0 .. i] ~ (*packs)[i+1 .. $];
512 		}
513 
514 		writeLocalPackageList(type);
515 
516 		foreach(ver, name; removed)
517 			logInfo("Deregistered package: %s (version: %s)", name, ver);
518 	}
519 
520 	/// For the given type add another path where packages will be looked up.
521 	void addSearchPath(NativePath path, LocalPackageType type)
522 	{
523 		m_repositories[type].searchPath ~= path;
524 		writeLocalPackageList(type);
525 	}
526 
527 	/// Removes a search path from the given type.
528 	void removeSearchPath(NativePath path, LocalPackageType type)
529 	{
530 		m_repositories[type].searchPath = m_repositories[type].searchPath.filter!(p => p != path)().array();
531 		writeLocalPackageList(type);
532 	}
533 
534 	void refresh(bool refresh_existing_packages)
535 	{
536 		logDiagnostic("Refreshing local packages (refresh existing: %s)...", refresh_existing_packages);
537 
538 		// load locally defined packages
539 		void scanLocalPackages(LocalPackageType type)
540 		{
541 			NativePath list_path = m_repositories[type].packagePath;
542 			Package[] packs;
543 			NativePath[] paths;
544 			if (!m_disableDefaultSearchPaths) try {
545 				auto local_package_file = list_path ~ LocalPackagesFilename;
546 				logDiagnostic("Looking for local package map at %s", local_package_file.toNativeString());
547 				if( !existsFile(local_package_file) ) return;
548 				logDiagnostic("Try to load local package map at %s", local_package_file.toNativeString());
549 				auto packlist = jsonFromFile(list_path ~ LocalPackagesFilename);
550 				enforce(packlist.type == Json.Type.array, LocalPackagesFilename~" must contain an array.");
551 				foreach( pentry; packlist ){
552 					try {
553 						auto name = pentry["name"].get!string;
554 						auto path = NativePath(pentry["path"].get!string);
555 						if (name == "*") {
556 							paths ~= path;
557 						} else {
558 							auto ver = Version(pentry["version"].get!string);
559 
560 							Package pp;
561 							if (!refresh_existing_packages) {
562 								foreach (p; m_repositories[type].localPackages)
563 									if (p.path == path) {
564 										pp = p;
565 										break;
566 									}
567 							}
568 
569 							if (!pp) {
570 								auto infoFile = Package.findPackageFile(path);
571 								if (!infoFile.empty) pp = Package.load(path, infoFile);
572 								else {
573 									logWarn("Locally registered package %s %s was not found. Please run 'dub remove-local \"%s\"'.",
574 										name, ver, path.toNativeString());
575 									auto info = Json.emptyObject;
576 									info["name"] = name;
577 									pp = new Package(info, path);
578 								}
579 							}
580 
581 							if (pp.name != name)
582 								logWarn("Local package at %s has different name than %s (%s)", path.toNativeString(), name, pp.name);
583 							pp.version_ = ver;
584 
585 							addPackages(packs, pp);
586 						}
587 					} catch( Exception e ){
588 						logWarn("Error adding local package: %s", e.msg);
589 					}
590 				}
591 			} catch( Exception e ){
592 				logDiagnostic("Loading of local package list at %s failed: %s", list_path.toNativeString(), e.msg);
593 			}
594 			m_repositories[type].localPackages = packs;
595 			m_repositories[type].searchPath = paths;
596 		}
597 		scanLocalPackages(LocalPackageType.system);
598 		scanLocalPackages(LocalPackageType.user);
599 
600 		auto old_packages = m_packages;
601 
602 		// rescan the system and user package folder
603 		void scanPackageFolder(NativePath path)
604 		{
605 			if( path.existsDirectory() ){
606 				logDebug("iterating dir %s", path.toNativeString());
607 				try foreach( pdir; iterateDirectory(path) ){
608 					logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
609 					if (!pdir.isDirectory) continue;
610 
611 					auto pack_path = path ~ (pdir.name ~ "/");
612 
613 					auto packageFile = Package.findPackageFile(pack_path);
614 
615 					if (isManagedPath(path) && packageFile.empty) {
616 						// Search for a single directory within this directory which happen to be a prefix of pdir
617 						// This is to support new folder structure installed over the ancient one.
618 						foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/")))
619 							if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) {// eg: package vibe-d will be in "vibe-d-x.y.z/vibe-d"
620 								pack_path ~= subdir.name ~ "/";
621 								packageFile = Package.findPackageFile(pack_path);
622 								break;
623 							}
624 					}
625 
626 					if (packageFile.empty) continue;
627 					Package p;
628 					try {
629 						if (!refresh_existing_packages)
630 							foreach (pp; old_packages)
631 								if (pp.path == pack_path) {
632 									p = pp;
633 									break;
634 								}
635 						if (!p) p = Package.load(pack_path, packageFile);
636 						addPackages(m_packages, p);
637 					} catch( Exception e ){
638 						logError("Failed to load package in %s: %s", pack_path, e.msg);
639 						logDiagnostic("Full error: %s", e.toString().sanitize());
640 					}
641 				}
642 				catch(Exception e) logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString());
643 			}
644 		}
645 
646 		m_packages = null;
647 		foreach (p; this.completeSearchPath)
648 			scanPackageFolder(p);
649 
650 		void loadOverrides(LocalPackageType type)
651 		{
652 			m_repositories[type].overrides = null;
653 			auto ovrfilepath = m_repositories[type].packagePath ~ LocalOverridesFilename;
654 			if (existsFile(ovrfilepath)) {
655 				foreach (entry; jsonFromFile(ovrfilepath)) {
656 					PackageOverride ovr;
657 					ovr.package_ = entry["name"].get!string;
658 					ovr.version_ = Dependency(entry["version"].get!string);
659 					if (auto pv = "targetVersion" in entry) ovr.targetVersion = Version(pv.get!string);
660 					if (auto pv = "targetPath" in entry) ovr.targetPath = NativePath(pv.get!string);
661 					m_repositories[type].overrides ~= ovr;
662 				}
663 			}
664 		}
665 		loadOverrides(LocalPackageType.user);
666 		loadOverrides(LocalPackageType.system);
667 	}
668 
669 	alias Hash = ubyte[];
670 	/// Generates a hash value for a given package.
671 	/// Some files or folders are ignored during the generation (like .dub and
672 	/// .svn folders)
673 	Hash hashPackage(Package pack)
674 	{
675 		string[] ignored_directories = [".git", ".dub", ".svn"];
676 		// something from .dub_ignore or what?
677 		string[] ignored_files = [];
678 		SHA1 sha1;
679 		foreach(file; dirEntries(pack.path.toNativeString(), SpanMode.depth)) {
680 			if(file.isDir && ignored_directories.canFind(NativePath(file.name).head.toString()))
681 				continue;
682 			else if(ignored_files.canFind(NativePath(file.name).head.toString()))
683 				continue;
684 
685 			sha1.put(cast(ubyte[])NativePath(file.name).head.toString());
686 			if(file.isDir) {
687 				logDebug("Hashed directory name %s", NativePath(file.name).head);
688 			}
689 			else {
690 				sha1.put(openFile(NativePath(file.name)).readAll());
691 				logDebug("Hashed file contents from %s", NativePath(file.name).head);
692 			}
693 		}
694 		auto hash = sha1.finish();
695 		logDebug("Project hash: %s", hash);
696 		return hash[].dup;
697 	}
698 
699 	private void writeLocalPackageList(LocalPackageType type)
700 	{
701 		Json[] newlist;
702 		foreach (p; m_repositories[type].searchPath) {
703 			auto entry = Json.emptyObject;
704 			entry["name"] = "*";
705 			entry["path"] = p.toNativeString();
706 			newlist ~= entry;
707 		}
708 
709 		foreach (p; m_repositories[type].localPackages) {
710 			if (p.parentPackage) continue; // do not store sub packages
711 			auto entry = Json.emptyObject;
712 			entry["name"] = p.name;
713 			entry["version"] = p.version_.toString();
714 			entry["path"] = p.path.toNativeString();
715 			newlist ~= entry;
716 		}
717 
718 		NativePath path = m_repositories[type].packagePath;
719 		if( !existsDirectory(path) ) mkdirRecurse(path.toNativeString());
720 		writeJsonFile(path ~ LocalPackagesFilename, Json(newlist));
721 	}
722 
723 	private void writeLocalPackageOverridesFile(LocalPackageType type)
724 	{
725 		Json[] newlist;
726 		foreach (ovr; m_repositories[type].overrides) {
727 			auto jovr = Json.emptyObject;
728 			jovr["name"] = ovr.package_;
729 			jovr["version"] = ovr.version_.versionSpec;
730 			if (!ovr.targetPath.empty) jovr["targetPath"] = ovr.targetPath.toNativeString();
731 			else jovr["targetVersion"] = ovr.targetVersion.toString();
732 			newlist ~= jovr;
733 		}
734 		auto path = m_repositories[type].packagePath;
735 		if (!existsDirectory(path)) mkdirRecurse(path.toNativeString());
736 		writeJsonFile(path ~ LocalOverridesFilename, Json(newlist));
737 	}
738 
739 	/// Adds the package and scans for subpackages.
740 	private void addPackages(ref Package[] dst_repos, Package pack)
741 	const {
742 		// Add the main package.
743 		dst_repos ~= pack;
744 
745 		// Additionally to the internally defined subpackages, whose metadata
746 		// is loaded with the main dub.json, load all externally defined
747 		// packages after the package is available with all the data.
748 		foreach (spr; pack.subPackages) {
749 			Package sp;
750 
751 			if (spr.path.length) {
752 				auto p = NativePath(spr.path);
753 				p.normalize();
754 				enforce(!p.absolute, "Sub package paths must be sub paths of the parent package.");
755 				auto path = pack.path ~ p;
756 				if (!existsFile(path)) {
757 					logError("Package %s declared a sub-package, definition file is missing: %s", pack.name, path.toNativeString());
758 					continue;
759 				}
760 				sp = Package.load(path, NativePath.init, pack);
761 			} else sp = new Package(spr.recipe, pack.path, pack);
762 
763 			// Add the subpackage.
764 			try {
765 				dst_repos ~= sp;
766 			} catch (Exception e) {
767 				logError("Package '%s': Failed to load sub-package %s: %s", pack.name,
768 					spr.path.length ? spr.path : spr.recipe.name, e.msg);
769 				logDiagnostic("Full error: %s", e.toString().sanitize());
770 			}
771 		}
772 	}
773 }
774 
775 struct PackageOverride {
776 	string package_;
777 	Dependency version_;
778 	Version targetVersion;
779 	NativePath targetPath;
780 
781 	this(string package_, Dependency version_, Version target_version)
782 	{
783 		this.package_ = package_;
784 		this.version_ = version_;
785 		this.targetVersion = target_version;
786 	}
787 
788 	this(string package_, Dependency version_, NativePath target_path)
789 	{
790 		this.package_ = package_;
791 		this.version_ = version_;
792 		this.targetPath = target_path;
793 	}
794 }
795 
796 enum LocalPackageType {
797 	user,
798 	system
799 }
800 
801 private enum LocalPackagesFilename = "local-packages.json";
802 private enum LocalOverridesFilename = "local-overrides.json";
803 
804 
805 private struct Repository {
806 	NativePath path;
807 	NativePath packagePath;
808 	NativePath[] searchPath;
809 	Package[] localPackages;
810 	PackageOverride[] overrides;
811 
812 	this(NativePath path)
813 	{
814 		this.path = path;
815 		this.packagePath = path ~"packages/";
816 	}
817 }