1 /**
2 	Stuff with dependencies.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff
7 */
8 module dub.package_;
9 
10 public import dub.recipe.packagerecipe;
11 
12 import dub.compilers.compiler;
13 import dub.dependency;
14 import dub.recipe.json;
15 import dub.recipe.sdl;
16 
17 import dub.internal.utils;
18 import dub.internal.vibecompat.core.log;
19 import dub.internal.vibecompat.core.file;
20 import dub.internal.vibecompat.data.json;
21 import dub.internal.vibecompat.inet.url;
22 
23 import std.algorithm;
24 import std.array;
25 import std.conv;
26 import std.exception;
27 import std.file;
28 import std.range;
29 import std.string;
30 
31 
32 
33 enum PackageFormat { json, sdl }
34 struct FilenameAndFormat
35 {
36 	string filename;
37 	PackageFormat format;
38 }
39 struct PathAndFormat
40 {
41 	Path path;
42 	PackageFormat format;
43 	@property bool empty() { return path.empty; }
44 	string toString() { return path.toString(); }
45 }
46 
47 // Supported package descriptions in decreasing order of preference.
48 static immutable FilenameAndFormat[] packageInfoFiles = [
49 	{"dub.json", PackageFormat.json},
50 	/*{"dub.sdl",PackageFormat.sdl},*/
51 	{"package.json", PackageFormat.json}
52 ];
53 
54 @property string[] packageInfoFilenames() { return packageInfoFiles.map!(f => cast(string)f.filename).array; }
55 
56 @property string defaultPackageFilename() { return packageInfoFiles[0].filename; }
57 
58 
59 /**
60 	Represents a package, including its sub packages
61 
62 	Documentation of the dub.json can be found at
63 	http://registry.vibed.org/package-format
64 */
65 class Package {
66 	private {
67 		Path m_path;
68 		PathAndFormat m_infoFile;
69 		PackageRecipe m_info;
70 		Package m_parentPackage;
71 	}
72 
73 	static PathAndFormat findPackageFile(Path path)
74 	{
75 		foreach(file; packageInfoFiles) {
76 			auto filename = path ~ file.filename;
77 			if(existsFile(filename)) return PathAndFormat(filename, file.format);
78 		}
79 		return PathAndFormat(Path());
80 	}
81 
82 	this(Path root, PathAndFormat infoFile = PathAndFormat(), Package parent = null, string versionOverride = "")
83 	{
84 		RawPackage raw_package;
85 		m_infoFile = infoFile;
86 
87 		try {
88 			if(m_infoFile.empty) {
89 				m_infoFile = findPackageFile(root);
90 				if(m_infoFile.empty) throw new Exception("no package file was found, expected one of the following: "~to!string(packageInfoFiles));
91 			}
92 			raw_package = rawPackageFromFile(m_infoFile);
93 		} catch (Exception ex) throw ex;//throw new Exception(format("Failed to load package %s: %s", m_infoFile.toNativeString(), ex.msg));
94 
95 		enforce(raw_package !is null, format("Missing package description for package at %s", root.toNativeString()));
96 		this(raw_package, root, parent, versionOverride);
97 	}
98 
99 	this(Json package_info, Path root = Path(), Package parent = null, string versionOverride = "")
100 	{
101 		this(new JsonPackage(package_info), root, parent, versionOverride);
102 	}
103 
104 	this(RawPackage raw_package, Path root = Path(), Package parent = null, string versionOverride = "")
105 	{
106 		PackageRecipe recipe;
107 
108 		// parse the Package description
109 		if(raw_package !is null)
110 		{
111 			scope(failure) logError("Failed to parse package description for %s %s in %s.",
112 				raw_package.package_name, versionOverride.length ? versionOverride : raw_package.version_,
113 				root.length ? root.toNativeString() : "remote location");
114 			raw_package.parseInto(recipe, parent ? parent.name : null);
115 
116 			if (!versionOverride.empty)
117 				recipe.version_ = versionOverride;
118 
119 			// try to run git to determine the version of the package if no explicit version was given
120 			if (recipe.version_.length == 0 && !parent) {
121 				try recipe.version_ = determineVersionFromSCM(root);
122 				catch (Exception e) logDebug("Failed to determine version by SCM: %s", e.msg);
123 
124 				if (recipe.version_.length == 0) {
125 					logDiagnostic("Note: Failed to determine version of package %s at %s. Assuming ~master.", recipe.name, this.path.toNativeString());
126 					// TODO: Assume unknown version here?
127 					// recipe.version_ = Version.UNKNOWN.toString();
128 					recipe.version_ = Version.MASTER.toString();
129 				} else logDiagnostic("Determined package version using GIT: %s %s", recipe.name, recipe.version_);
130 			}
131 		}
132 
133 		this(recipe, root, parent);
134 	}
135 
136 	this(PackageRecipe recipe, Path root = Path(), Package parent = null)
137 	{
138 		m_parentPackage = parent;
139 		m_path = root;
140 		m_path.endsWithSlash = true;
141 
142 		// use the given recipe as the basis
143 		m_info = recipe;
144 
145 		fillWithDefaults();
146 		simpleLint();
147 	}
148 
149 	@property string name()
150 	const {
151 		if (m_parentPackage) return m_parentPackage.name ~ ":" ~ m_info.name;
152 		else return m_info.name;
153 	}
154 	@property string vers() const { return m_parentPackage ? m_parentPackage.vers : m_info.version_; }
155 	@property Version ver() const { return Version(this.vers); }
156 	@property void ver(Version ver) { assert(m_parentPackage is null); m_info.version_ = ver.toString(); }
157 	@property ref inout(PackageRecipe) info() inout { return m_info; }
158 	@property Path path() const { return m_path; }
159 	@property Path packageInfoFilename() const { return m_infoFile.path; }
160 	@property const(Dependency[string]) dependencies() const { return m_info.dependencies; }
161 	@property inout(Package) basePackage() inout { return m_parentPackage ? m_parentPackage.basePackage : this; }
162 	@property inout(Package) parentPackage() inout { return m_parentPackage; }
163 	@property inout(SubPackage)[] subPackages() inout { return m_info.subPackages; }
164 
165 	@property string[] configurations()
166 	const {
167 		auto ret = appender!(string[])();
168 		foreach( ref config; m_info.configurations )
169 			ret.put(config.name);
170 		return ret.data;
171 	}
172 
173 	const(Dependency[string]) getDependencies(string config)
174 	const {
175 		Dependency[string] ret;
176 		foreach (k, v; m_info.buildSettings.dependencies)
177 			ret[k] = v;
178 		foreach (ref conf; m_info.configurations)
179 			if (conf.name == config) {
180 				foreach (k, v; conf.buildSettings.dependencies)
181 					ret[k] = v;
182 				break;
183 			}
184 		return ret;
185 	}
186 
187 	/** Overwrites the packge description file using the default filename with the current information.
188 	*/
189 	void storeInfo()
190 	{
191 		enforce(!ver.isUnknown, "Trying to store a package with an 'unknown' version, this is not supported.");
192 		auto filename = m_path ~ defaultPackageFilename;
193 		auto dstFile = openFile(filename.toNativeString(), FileMode.CreateTrunc);
194 		scope(exit) dstFile.close();
195 		dstFile.writePrettyJsonString(m_info.toJson());
196 		m_infoFile = PathAndFormat(filename);
197 	}
198 
199 	/*inout(Package) getSubPackage(string name, bool silent_fail = false)
200 	inout {
201 		foreach (p; m_info.subPackages)
202 			if (p.package_ !is null && p.package_.name == this.name ~ ":" ~ name)
203 				return p.package_;
204 		enforce(silent_fail, format("Unknown sub package: %s:%s", this.name, name));
205 		return null;
206 	}*/
207 
208 	void warnOnSpecialCompilerFlags()
209 	{
210 		// warn about use of special flags
211 		m_info.buildSettings.warnOnSpecialCompilerFlags(m_info.name, null);
212 		foreach (ref config; m_info.configurations)
213 			config.buildSettings.warnOnSpecialCompilerFlags(m_info.name, config.name);
214 	}
215 
216 	const(BuildSettingsTemplate) getBuildSettings(string config = null)
217 	const {
218 		if (config.length) {
219 			foreach (ref conf; m_info.configurations)
220 				if (conf.name == config)
221 					return conf.buildSettings;
222 			assert(false, "Unknown configuration: "~config);
223 		} else {
224 			return m_info.buildSettings;
225 		}
226 	}
227 
228 	/// Returns all BuildSettings for the given platform and config.
229 	BuildSettings getBuildSettings(in BuildPlatform platform, string config)
230 	const {
231 		BuildSettings ret;
232 		m_info.buildSettings.getPlatformSettings(ret, platform, this.path);
233 		bool found = false;
234 		foreach(ref conf; m_info.configurations){
235 			if( conf.name != config ) continue;
236 			conf.buildSettings.getPlatformSettings(ret, platform, this.path);
237 			found = true;
238 			break;
239 		}
240 		assert(found || config is null, "Unknown configuration for "~m_info.name~": "~config);
241 
242 		// construct default target name based on package name
243 		if( ret.targetName.empty ) ret.targetName = this.name.replace(":", "_");
244 
245 		// special support for DMD style flags
246 		getCompiler("dmd").extractBuildOptions(ret);
247 
248 		return ret;
249 	}
250 
251 	/// Returns the combination of all build settings for all configurations and platforms
252 	BuildSettings getCombinedBuildSettings()
253 	const {
254 		BuildSettings ret;
255 		m_info.buildSettings.getPlatformSettings(ret, BuildPlatform.any, this.path);
256 		foreach(ref conf; m_info.configurations)
257 			conf.buildSettings.getPlatformSettings(ret, BuildPlatform.any, this.path);
258 
259 		// construct default target name based on package name
260 		if (ret.targetName.empty) ret.targetName = this.name.replace(":", "_");
261 
262 		// special support for DMD style flags
263 		getCompiler("dmd").extractBuildOptions(ret);
264 
265 		return ret;
266 	}
267 
268 	void addBuildTypeSettings(ref BuildSettings settings, in BuildPlatform platform, string build_type)
269 	const {
270 		if (build_type == "$DFLAGS") {
271 			import std.process;
272 			string dflags = environment.get("DFLAGS");
273 			settings.addDFlags(dflags.split());
274 			return;
275 		}
276 
277 		if (auto pbt = build_type in m_info.buildTypes) {
278 			logDiagnostic("Using custom build type '%s'.", build_type);
279 			pbt.getPlatformSettings(settings, platform, this.path);
280 		} else {
281 			with(BuildOptions) switch (build_type) {
282 				default: throw new Exception(format("Unknown build type for %s: '%s'", this.name, build_type));
283 				case "plain": break;
284 				case "debug": settings.addOptions(debugMode, debugInfo); break;
285 				case "release": settings.addOptions(releaseMode, optimize, inline); break;
286 				case "release-nobounds": settings.addOptions(releaseMode, optimize, inline, noBoundsCheck); break;
287 				case "unittest": settings.addOptions(unittests, debugMode, debugInfo); break;
288 				case "docs": settings.addOptions(syntaxOnly); settings.addDFlags("-c", "-Dddocs"); break;
289 				case "ddox": settings.addOptions(syntaxOnly); settings.addDFlags("-c", "-Df__dummy.html", "-Xfdocs.json"); break;
290 				case "profile": settings.addOptions(profile, optimize, inline, debugInfo); break;
291 				case "cov": settings.addOptions(coverage, debugInfo); break;
292 				case "unittest-cov": settings.addOptions(unittests, coverage, debugMode, debugInfo); break;
293 			}
294 		}
295 	}
296 
297 	string getSubConfiguration(string config, in Package dependency, in BuildPlatform platform)
298 	const {
299 		bool found = false;
300 		foreach(ref c; m_info.configurations){
301 			if( c.name == config ){
302 				if( auto pv = dependency.name in c.buildSettings.subConfigurations ) return *pv;
303 				found = true;
304 				break;
305 			}
306 		}
307 		assert(found || config is null, "Invalid configuration \""~config~"\" for "~this.name);
308 		if( auto pv = dependency.name in m_info.buildSettings.subConfigurations ) return *pv;
309 		return null;
310 	}
311 
312 	/// Returns the default configuration to build for the given platform
313 	string getDefaultConfiguration(in BuildPlatform platform, bool allow_non_library = false)
314 	const {
315 		foreach (ref conf; m_info.configurations) {
316 			if (!conf.matchesPlatform(platform)) continue;
317 			if (!allow_non_library && conf.buildSettings.targetType == TargetType.executable) continue;
318 			return conf.name;
319 		}
320 		return null;
321 	}
322 
323 	/// Returns a list of configurations suitable for the given platform
324 	string[] getPlatformConfigurations(in BuildPlatform platform, bool is_main_package = false)
325 	const {
326 		auto ret = appender!(string[]);
327 		foreach(ref conf; m_info.configurations){
328 			if (!conf.matchesPlatform(platform)) continue;
329 			if (!is_main_package && conf.buildSettings.targetType == TargetType.executable) continue;
330 			ret ~= conf.name;
331 		}
332 		if (ret.data.length == 0) ret.put(null);
333 		return ret.data;
334 	}
335 
336 	/// Human readable information of this package and its dependencies.
337 	string generateInfoString() const {
338 		string s;
339 		s ~= m_info.name ~ ", version '" ~ m_info.version_ ~ "'";
340 		s ~= "\n  Dependencies:";
341 		foreach(string p, ref const Dependency v; m_info.dependencies)
342 			s ~= "\n    " ~ p ~ ", version '" ~ v.toString() ~ "'";
343 		return s;
344 	}
345 
346 	bool hasDependency(string depname, string config)
347 	const {
348 		if (depname in m_info.buildSettings.dependencies) return true;
349 		foreach (ref c; m_info.configurations)
350 			if ((config.empty || c.name == config) && depname in c.buildSettings.dependencies)
351 				return true;
352 		return false;
353 	}
354 
355 	void describe(ref Json dst, BuildPlatform platform, string config)
356 	{
357 		dst.path = m_path.toNativeString();
358 		dst.name = this.name;
359 		dst["version"] = this.vers;
360 		dst.description = m_info.description;
361 		dst.homepage = m_info.homepage;
362 		dst.authors = m_info.authors.serializeToJson();
363 		dst.copyright = m_info.copyright;
364 		dst.license = m_info.license;
365 		dst.dependencies = m_info.dependencies.keys.serializeToJson();
366 
367 		// save build settings
368 		BuildSettings bs = getBuildSettings(platform, config);
369 		BuildSettings allbs = getCombinedBuildSettings();
370 
371 		foreach (string k, v; bs.serializeToJson()) dst[k] = v;
372 		dst.remove("requirements");
373 		dst.remove("sourceFiles");
374 		dst.remove("importFiles");
375 		dst.remove("stringImportFiles");
376 		dst.targetType = bs.targetType.to!string();
377 		if (dst.targetType != TargetType.none)
378 			dst.targetFileName = getTargetFileName(bs, platform);
379 
380 		// prettify build requirements output
381 		Json[] breqs;
382 		for (int i = 1; i <= BuildRequirements.max; i <<= 1)
383 			if (bs.requirements & i)
384 				breqs ~= Json(to!string(cast(BuildRequirements)i));
385 		dst.buildRequirements = breqs;
386 
387 		// prettify options output
388 		Json[] bopts;
389 		for (int i = 1; i <= BuildOptions.max; i <<= 1)
390 			if (bs.options & i)
391 				bopts ~= Json(to!string(cast(BuildOptions)i));
392 		dst.options = bopts;
393 
394 		// collect all possible source files and determine their types
395 		string[string] sourceFileTypes;
396 		foreach (f; allbs.stringImportFiles) sourceFileTypes[f] = "unusedStringImport";
397 		foreach (f; allbs.importFiles) sourceFileTypes[f] = "unusedImport";
398 		foreach (f; allbs.sourceFiles) sourceFileTypes[f] = "unusedSource";
399 		foreach (f; bs.stringImportFiles) sourceFileTypes[f] = "stringImport";
400 		foreach (f; bs.importFiles) sourceFileTypes[f] = "import";
401 		foreach (f; bs.sourceFiles) sourceFileTypes[f] = "source";
402 		Json[] files;
403 		foreach (f; sourceFileTypes.byKey.array.sort()) {
404 			auto jf = Json.emptyObject;
405 			jf["path"] = f;
406 			jf["type"] = sourceFileTypes[f];
407 			files ~= jf;
408 		}
409 		dst.files = Json(files);
410 	}
411 
412 	private void fillWithDefaults()
413 	{
414 		auto bs = &m_info.buildSettings;
415 
416 		// check for default string import folders
417 		if ("" !in bs.stringImportPaths) {
418 			foreach(defvf; ["views"]){
419 				if( existsFile(m_path ~ defvf) )
420 					bs.stringImportPaths[""] ~= defvf;
421 			}
422 		}
423 
424 		// check for default source folders
425 		immutable hasSP = ("" in bs.sourcePaths) !is null;
426 		immutable hasIP = ("" in bs.importPaths) !is null;
427 		if (!hasSP || !hasIP) {
428 			foreach (defsf; ["source/", "src/"]) {
429 				if (existsFile(m_path ~ defsf)) {
430 					if (!hasSP) bs.sourcePaths[""] ~= defsf;
431 					if (!hasIP) bs.importPaths[""] ~= defsf;
432 				}
433 			}
434 		}
435 
436 		// check for default app_main
437 		string app_main_file;
438 		auto pkg_name = m_info.name.length ? m_info.name : "unknown";
439 		foreach(sf; bs.sourcePaths.get("", null)){
440 			auto p = m_path ~ sf;
441 			if( !existsFile(p) ) continue;
442 			foreach(fil; ["app.d", "main.d", pkg_name ~ "/main.d", pkg_name ~ "/" ~ "app.d"]){
443 				if( existsFile(p ~ fil) ) {
444 					app_main_file = (Path(sf) ~ fil).toNativeString();
445 					break;
446 				}
447 			}
448 		}
449 
450 		// generate default configurations if none are defined
451 		if (m_info.configurations.length == 0) {
452 			if (bs.targetType == TargetType.executable) {
453 				BuildSettingsTemplate app_settings;
454 				app_settings.targetType = TargetType.executable;
455 				if (bs.mainSourceFile.empty) app_settings.mainSourceFile = app_main_file;
456 				m_info.configurations ~= ConfigurationInfo("application", app_settings);
457 			} else if (bs.targetType != TargetType.none) {
458 				BuildSettingsTemplate lib_settings;
459 				lib_settings.targetType = bs.targetType == TargetType.autodetect ? TargetType.library : bs.targetType;
460 
461 				if (bs.targetType == TargetType.autodetect) {
462 					if (app_main_file.length) {
463 						lib_settings.excludedSourceFiles[""] ~= app_main_file;
464 
465 						BuildSettingsTemplate app_settings;
466 						app_settings.targetType = TargetType.executable;
467 						app_settings.mainSourceFile = app_main_file;
468 						m_info.configurations ~= ConfigurationInfo("application", app_settings);
469 					}
470 				}
471 
472 				m_info.configurations ~= ConfigurationInfo("library", lib_settings);
473 			}
474 		}
475 	}
476 
477 	private void simpleLint() const {
478 		if (m_parentPackage) {
479 			if (m_parentPackage.path != path) {
480 				if (info.license.length && info.license != m_parentPackage.info.license)
481 					logWarn("License in subpackage %s is different than it's parent package, this is discouraged.", name);
482 			}
483 		}
484 		if (name.empty) logWarn("The package in %s has no name.", path);
485 	}
486 
487 	private static RawPackage rawPackageFromFile(PathAndFormat file, bool silent_fail = false) {
488 		if( silent_fail && !existsFile(file.path) ) return null;
489 
490 		string text;
491 
492 		{
493 			auto f = openFile(file.path.toNativeString(), FileMode.Read);
494 			scope(exit) f.close();
495 			text = stripUTF8Bom(cast(string)f.readAll());
496 		}
497 
498 		final switch(file.format) {
499 			case PackageFormat.json:
500 				return new JsonPackage(parseJsonString(text, file.path.toNativeString()));
501 			case PackageFormat.sdl:
502 				if(silent_fail) return null; throw new Exception("SDL not implemented");
503 		}
504 	}
505 
506 	static abstract class RawPackage
507 	{
508 		string package_name; // Should already be lower case
509 		string version_;
510 		abstract void parseInto(ref PackageRecipe package_, string parent_name);
511 	}
512 	private static class JsonPackage : RawPackage
513 	{
514 		Json json;
515 		this(Json json) {
516 			this.json = json;
517 
518 			string nameLower;
519 			if(json.type == Json.Type..string) {
520 				nameLower = json.get!string.toLower();
521 				this.json = nameLower;
522 			} else {
523 				nameLower = json.name.get!string.toLower();
524 				this.json.name = nameLower;
525 				this.package_name = nameLower;
526 
527 				Json versionJson = json["version"];
528 				this.version_ = (versionJson.type == Json.Type.undefined) ? null : versionJson.get!string;
529 			}
530 
531 			this.package_name = nameLower;
532 		}
533 		override void parseInto(ref PackageRecipe recipe, string parent_name)
534 		{
535 			recipe.parseJson(json, parent_name);
536 		}
537 	}
538 	private static class SdlPackage : RawPackage
539 	{
540 		override void parseInto(ref PackageRecipe package_, string parent_name)
541 		{
542 			throw new Exception("SDL packages not implemented yet");
543 		}
544 	}
545 }
546 
547 
548 private string determineVersionFromSCM(Path path)
549 {
550 	import std.process;
551 	import dub.semver;
552 
553 	auto git_dir = path ~ ".git";
554 	if (!existsFile(git_dir) || !isDir(git_dir.toNativeString)) return null;
555 	auto git_dir_param = "--git-dir=" ~ git_dir.toNativeString();
556 
557 	static string exec(scope string[] params...) {
558 		auto ret = executeShell(escapeShellCommand(params));
559 		if (ret.status == 0) return ret.output.strip;
560 		logDebug("'%s' failed with exit code %s: %s", params.join(" "), ret.status, ret.output.strip);
561 		return null;
562 	}
563 
564 	if (auto tag = exec("git", git_dir_param, "describe", "--long", "--tags")) {
565 		auto parts = tag.split("-");
566 		auto commit = parts[$-1];
567 		auto num = parts[$-2].to!int;
568 		tag = parts[0 .. $-2].join("-");
569 		if (tag.startsWith("v") && isValidVersion(tag[1 .. $])) {
570 			if (num == 0) return tag[1 .. $];
571 			else if (tag.canFind("+")) return format("%s.commit.%s.%s", tag[1 .. $], num, commit);
572 			else return format("%s+commit.%s.%s", tag[1 .. $], num, commit);
573 		}
574 	}
575 
576 	if (auto branch = exec("git", git_dir_param, "rev-parse", "--abbrev-ref", "HEAD")) {
577 		if (branch != "HEAD") return "~" ~ branch;
578 	}
579 
580 	return null;
581 }