1 /** 2 Stuff with dependencies. 3 4 Copyright: © 2012-2013 Matthias Dondorff 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Matthias Dondorff 7 */ 8 module dub.package_; 9 10 public import dub.recipe.packagerecipe; 11 12 import dub.compilers.compiler; 13 import dub.dependency; 14 import dub.recipe.json; 15 import dub.recipe.sdl; 16 17 import dub.internal.utils; 18 import dub.internal.vibecompat.core.log; 19 import dub.internal.vibecompat.core.file; 20 import dub.internal.vibecompat.data.json; 21 import dub.internal.vibecompat.inet.url; 22 23 import std.algorithm; 24 import std.array; 25 import std.conv; 26 import std.exception; 27 import std.file; 28 import std.range; 29 import std.string; 30 31 32 33 enum PackageFormat { json, sdl } 34 struct FilenameAndFormat 35 { 36 string filename; 37 PackageFormat format; 38 } 39 struct PathAndFormat 40 { 41 Path path; 42 PackageFormat format; 43 @property bool empty() { return path.empty; } 44 string toString() { return path.toString(); } 45 } 46 47 // Supported package descriptions in decreasing order of preference. 48 static immutable FilenameAndFormat[] packageInfoFiles = [ 49 {"dub.json", PackageFormat.json}, 50 /*{"dub.sdl",PackageFormat.sdl},*/ 51 {"package.json", PackageFormat.json} 52 ]; 53 54 @property string[] packageInfoFilenames() { return packageInfoFiles.map!(f => cast(string)f.filename).array; } 55 56 @property string defaultPackageFilename() { return packageInfoFiles[0].filename; } 57 58 59 /** 60 Represents a package, including its sub packages 61 62 Documentation of the dub.json can be found at 63 http://registry.vibed.org/package-format 64 */ 65 class Package { 66 private { 67 Path m_path; 68 PathAndFormat m_infoFile; 69 PackageRecipe m_info; 70 Package m_parentPackage; 71 } 72 73 static PathAndFormat findPackageFile(Path path) 74 { 75 foreach(file; packageInfoFiles) { 76 auto filename = path ~ file.filename; 77 if(existsFile(filename)) return PathAndFormat(filename, file.format); 78 } 79 return PathAndFormat(Path()); 80 } 81 82 this(Path root, PathAndFormat infoFile = PathAndFormat(), Package parent = null, string versionOverride = "") 83 { 84 RawPackage raw_package; 85 m_infoFile = infoFile; 86 87 try { 88 if(m_infoFile.empty) { 89 m_infoFile = findPackageFile(root); 90 if(m_infoFile.empty) throw new Exception("no package file was found, expected one of the following: "~to!string(packageInfoFiles)); 91 } 92 raw_package = rawPackageFromFile(m_infoFile); 93 } catch (Exception ex) throw ex;//throw new Exception(format("Failed to load package %s: %s", m_infoFile.toNativeString(), ex.msg)); 94 95 enforce(raw_package !is null, format("Missing package description for package at %s", root.toNativeString())); 96 this(raw_package, root, parent, versionOverride); 97 } 98 99 this(Json package_info, Path root = Path(), Package parent = null, string versionOverride = "") 100 { 101 this(new JsonPackage(package_info), root, parent, versionOverride); 102 } 103 104 this(RawPackage raw_package, Path root = Path(), Package parent = null, string versionOverride = "") 105 { 106 PackageRecipe recipe; 107 108 // parse the Package description 109 if(raw_package !is null) 110 { 111 scope(failure) logError("Failed to parse package description for %s %s in %s.", 112 raw_package.package_name, versionOverride.length ? versionOverride : raw_package.version_, 113 root.length ? root.toNativeString() : "remote location"); 114 raw_package.parseInto(recipe, parent ? parent.name : null); 115 116 if (!versionOverride.empty) 117 recipe.version_ = versionOverride; 118 119 // try to run git to determine the version of the package if no explicit version was given 120 if (recipe.version_.length == 0 && !parent) { 121 try recipe.version_ = determineVersionFromSCM(root); 122 catch (Exception e) logDebug("Failed to determine version by SCM: %s", e.msg); 123 124 if (recipe.version_.length == 0) { 125 logDiagnostic("Note: Failed to determine version of package %s at %s. Assuming ~master.", recipe.name, this.path.toNativeString()); 126 // TODO: Assume unknown version here? 127 // recipe.version_ = Version.UNKNOWN.toString(); 128 recipe.version_ = Version.MASTER.toString(); 129 } else logDiagnostic("Determined package version using GIT: %s %s", recipe.name, recipe.version_); 130 } 131 } 132 133 this(recipe, root, parent); 134 } 135 136 this(PackageRecipe recipe, Path root = Path(), Package parent = null) 137 { 138 m_parentPackage = parent; 139 m_path = root; 140 m_path.endsWithSlash = true; 141 142 // use the given recipe as the basis 143 m_info = recipe; 144 145 fillWithDefaults(); 146 simpleLint(); 147 } 148 149 @property string name() 150 const { 151 if (m_parentPackage) return m_parentPackage.name ~ ":" ~ m_info.name; 152 else return m_info.name; 153 } 154 @property string vers() const { return m_parentPackage ? m_parentPackage.vers : m_info.version_; } 155 @property Version ver() const { return Version(this.vers); } 156 @property void ver(Version ver) { assert(m_parentPackage is null); m_info.version_ = ver.toString(); } 157 @property ref inout(PackageRecipe) info() inout { return m_info; } 158 @property Path path() const { return m_path; } 159 @property Path packageInfoFilename() const { return m_infoFile.path; } 160 @property const(Dependency[string]) dependencies() const { return m_info.dependencies; } 161 @property inout(Package) basePackage() inout { return m_parentPackage ? m_parentPackage.basePackage : this; } 162 @property inout(Package) parentPackage() inout { return m_parentPackage; } 163 @property inout(SubPackage)[] subPackages() inout { return m_info.subPackages; } 164 165 @property string[] configurations() 166 const { 167 auto ret = appender!(string[])(); 168 foreach( ref config; m_info.configurations ) 169 ret.put(config.name); 170 return ret.data; 171 } 172 173 const(Dependency[string]) getDependencies(string config) 174 const { 175 Dependency[string] ret; 176 foreach (k, v; m_info.buildSettings.dependencies) 177 ret[k] = v; 178 foreach (ref conf; m_info.configurations) 179 if (conf.name == config) { 180 foreach (k, v; conf.buildSettings.dependencies) 181 ret[k] = v; 182 break; 183 } 184 return ret; 185 } 186 187 /** Overwrites the packge description file using the default filename with the current information. 188 */ 189 void storeInfo() 190 { 191 enforce(!ver.isUnknown, "Trying to store a package with an 'unknown' version, this is not supported."); 192 auto filename = m_path ~ defaultPackageFilename; 193 auto dstFile = openFile(filename.toNativeString(), FileMode.CreateTrunc); 194 scope(exit) dstFile.close(); 195 dstFile.writePrettyJsonString(m_info.toJson()); 196 m_infoFile = PathAndFormat(filename); 197 } 198 199 /*inout(Package) getSubPackage(string name, bool silent_fail = false) 200 inout { 201 foreach (p; m_info.subPackages) 202 if (p.package_ !is null && p.package_.name == this.name ~ ":" ~ name) 203 return p.package_; 204 enforce(silent_fail, format("Unknown sub package: %s:%s", this.name, name)); 205 return null; 206 }*/ 207 208 void warnOnSpecialCompilerFlags() 209 { 210 // warn about use of special flags 211 m_info.buildSettings.warnOnSpecialCompilerFlags(m_info.name, null); 212 foreach (ref config; m_info.configurations) 213 config.buildSettings.warnOnSpecialCompilerFlags(m_info.name, config.name); 214 } 215 216 const(BuildSettingsTemplate) getBuildSettings(string config = null) 217 const { 218 if (config.length) { 219 foreach (ref conf; m_info.configurations) 220 if (conf.name == config) 221 return conf.buildSettings; 222 assert(false, "Unknown configuration: "~config); 223 } else { 224 return m_info.buildSettings; 225 } 226 } 227 228 /// Returns all BuildSettings for the given platform and config. 229 BuildSettings getBuildSettings(in BuildPlatform platform, string config) 230 const { 231 BuildSettings ret; 232 m_info.buildSettings.getPlatformSettings(ret, platform, this.path); 233 bool found = false; 234 foreach(ref conf; m_info.configurations){ 235 if( conf.name != config ) continue; 236 conf.buildSettings.getPlatformSettings(ret, platform, this.path); 237 found = true; 238 break; 239 } 240 assert(found || config is null, "Unknown configuration for "~m_info.name~": "~config); 241 242 // construct default target name based on package name 243 if( ret.targetName.empty ) ret.targetName = this.name.replace(":", "_"); 244 245 // special support for DMD style flags 246 getCompiler("dmd").extractBuildOptions(ret); 247 248 return ret; 249 } 250 251 /// Returns the combination of all build settings for all configurations and platforms 252 BuildSettings getCombinedBuildSettings() 253 const { 254 BuildSettings ret; 255 m_info.buildSettings.getPlatformSettings(ret, BuildPlatform.any, this.path); 256 foreach(ref conf; m_info.configurations) 257 conf.buildSettings.getPlatformSettings(ret, BuildPlatform.any, this.path); 258 259 // construct default target name based on package name 260 if (ret.targetName.empty) ret.targetName = this.name.replace(":", "_"); 261 262 // special support for DMD style flags 263 getCompiler("dmd").extractBuildOptions(ret); 264 265 return ret; 266 } 267 268 void addBuildTypeSettings(ref BuildSettings settings, in BuildPlatform platform, string build_type) 269 const { 270 if (build_type == "$DFLAGS") { 271 import std.process; 272 string dflags = environment.get("DFLAGS"); 273 settings.addDFlags(dflags.split()); 274 return; 275 } 276 277 if (auto pbt = build_type in m_info.buildTypes) { 278 logDiagnostic("Using custom build type '%s'.", build_type); 279 pbt.getPlatformSettings(settings, platform, this.path); 280 } else { 281 with(BuildOptions) switch (build_type) { 282 default: throw new Exception(format("Unknown build type for %s: '%s'", this.name, build_type)); 283 case "plain": break; 284 case "debug": settings.addOptions(debugMode, debugInfo); break; 285 case "release": settings.addOptions(releaseMode, optimize, inline); break; 286 case "release-nobounds": settings.addOptions(releaseMode, optimize, inline, noBoundsCheck); break; 287 case "unittest": settings.addOptions(unittests, debugMode, debugInfo); break; 288 case "docs": settings.addOptions(syntaxOnly); settings.addDFlags("-c", "-Dddocs"); break; 289 case "ddox": settings.addOptions(syntaxOnly); settings.addDFlags("-c", "-Df__dummy.html", "-Xfdocs.json"); break; 290 case "profile": settings.addOptions(profile, optimize, inline, debugInfo); break; 291 case "cov": settings.addOptions(coverage, debugInfo); break; 292 case "unittest-cov": settings.addOptions(unittests, coverage, debugMode, debugInfo); break; 293 } 294 } 295 } 296 297 string getSubConfiguration(string config, in Package dependency, in BuildPlatform platform) 298 const { 299 bool found = false; 300 foreach(ref c; m_info.configurations){ 301 if( c.name == config ){ 302 if( auto pv = dependency.name in c.buildSettings.subConfigurations ) return *pv; 303 found = true; 304 break; 305 } 306 } 307 assert(found || config is null, "Invalid configuration \""~config~"\" for "~this.name); 308 if( auto pv = dependency.name in m_info.buildSettings.subConfigurations ) return *pv; 309 return null; 310 } 311 312 /// Returns the default configuration to build for the given platform 313 string getDefaultConfiguration(in BuildPlatform platform, bool allow_non_library = false) 314 const { 315 foreach (ref conf; m_info.configurations) { 316 if (!conf.matchesPlatform(platform)) continue; 317 if (!allow_non_library && conf.buildSettings.targetType == TargetType.executable) continue; 318 return conf.name; 319 } 320 return null; 321 } 322 323 /// Returns a list of configurations suitable for the given platform 324 string[] getPlatformConfigurations(in BuildPlatform platform, bool is_main_package = false) 325 const { 326 auto ret = appender!(string[]); 327 foreach(ref conf; m_info.configurations){ 328 if (!conf.matchesPlatform(platform)) continue; 329 if (!is_main_package && conf.buildSettings.targetType == TargetType.executable) continue; 330 ret ~= conf.name; 331 } 332 if (ret.data.length == 0) ret.put(null); 333 return ret.data; 334 } 335 336 /// Human readable information of this package and its dependencies. 337 string generateInfoString() const { 338 string s; 339 s ~= m_info.name ~ ", version '" ~ m_info.version_ ~ "'"; 340 s ~= "\n Dependencies:"; 341 foreach(string p, ref const Dependency v; m_info.dependencies) 342 s ~= "\n " ~ p ~ ", version '" ~ v.toString() ~ "'"; 343 return s; 344 } 345 346 bool hasDependency(string depname, string config) 347 const { 348 if (depname in m_info.buildSettings.dependencies) return true; 349 foreach (ref c; m_info.configurations) 350 if ((config.empty || c.name == config) && depname in c.buildSettings.dependencies) 351 return true; 352 return false; 353 } 354 355 void describe(ref Json dst, BuildPlatform platform, string config) 356 { 357 dst.path = m_path.toNativeString(); 358 dst.name = this.name; 359 dst["version"] = this.vers; 360 dst.description = m_info.description; 361 dst.homepage = m_info.homepage; 362 dst.authors = m_info.authors.serializeToJson(); 363 dst.copyright = m_info.copyright; 364 dst.license = m_info.license; 365 dst.dependencies = m_info.dependencies.keys.serializeToJson(); 366 367 // save build settings 368 BuildSettings bs = getBuildSettings(platform, config); 369 BuildSettings allbs = getCombinedBuildSettings(); 370 371 foreach (string k, v; bs.serializeToJson()) dst[k] = v; 372 dst.remove("requirements"); 373 dst.remove("sourceFiles"); 374 dst.remove("importFiles"); 375 dst.remove("stringImportFiles"); 376 dst.targetType = bs.targetType.to!string(); 377 if (dst.targetType != TargetType.none) 378 dst.targetFileName = getTargetFileName(bs, platform); 379 380 // prettify build requirements output 381 Json[] breqs; 382 for (int i = 1; i <= BuildRequirements.max; i <<= 1) 383 if (bs.requirements & i) 384 breqs ~= Json(to!string(cast(BuildRequirements)i)); 385 dst.buildRequirements = breqs; 386 387 // prettify options output 388 Json[] bopts; 389 for (int i = 1; i <= BuildOptions.max; i <<= 1) 390 if (bs.options & i) 391 bopts ~= Json(to!string(cast(BuildOptions)i)); 392 dst.options = bopts; 393 394 // collect all possible source files and determine their types 395 string[string] sourceFileTypes; 396 foreach (f; allbs.stringImportFiles) sourceFileTypes[f] = "unusedStringImport"; 397 foreach (f; allbs.importFiles) sourceFileTypes[f] = "unusedImport"; 398 foreach (f; allbs.sourceFiles) sourceFileTypes[f] = "unusedSource"; 399 foreach (f; bs.stringImportFiles) sourceFileTypes[f] = "stringImport"; 400 foreach (f; bs.importFiles) sourceFileTypes[f] = "import"; 401 foreach (f; bs.sourceFiles) sourceFileTypes[f] = "source"; 402 Json[] files; 403 foreach (f; sourceFileTypes.byKey.array.sort()) { 404 auto jf = Json.emptyObject; 405 jf["path"] = f; 406 jf["type"] = sourceFileTypes[f]; 407 files ~= jf; 408 } 409 dst.files = Json(files); 410 } 411 412 private void fillWithDefaults() 413 { 414 auto bs = &m_info.buildSettings; 415 416 // check for default string import folders 417 if ("" !in bs.stringImportPaths) { 418 foreach(defvf; ["views"]){ 419 if( existsFile(m_path ~ defvf) ) 420 bs.stringImportPaths[""] ~= defvf; 421 } 422 } 423 424 // check for default source folders 425 immutable hasSP = ("" in bs.sourcePaths) !is null; 426 immutable hasIP = ("" in bs.importPaths) !is null; 427 if (!hasSP || !hasIP) { 428 foreach (defsf; ["source/", "src/"]) { 429 if (existsFile(m_path ~ defsf)) { 430 if (!hasSP) bs.sourcePaths[""] ~= defsf; 431 if (!hasIP) bs.importPaths[""] ~= defsf; 432 } 433 } 434 } 435 436 // check for default app_main 437 string app_main_file; 438 auto pkg_name = m_info.name.length ? m_info.name : "unknown"; 439 foreach(sf; bs.sourcePaths.get("", null)){ 440 auto p = m_path ~ sf; 441 if( !existsFile(p) ) continue; 442 foreach(fil; ["app.d", "main.d", pkg_name ~ "/main.d", pkg_name ~ "/" ~ "app.d"]){ 443 if( existsFile(p ~ fil) ) { 444 app_main_file = (Path(sf) ~ fil).toNativeString(); 445 break; 446 } 447 } 448 } 449 450 // generate default configurations if none are defined 451 if (m_info.configurations.length == 0) { 452 if (bs.targetType == TargetType.executable) { 453 BuildSettingsTemplate app_settings; 454 app_settings.targetType = TargetType.executable; 455 if (bs.mainSourceFile.empty) app_settings.mainSourceFile = app_main_file; 456 m_info.configurations ~= ConfigurationInfo("application", app_settings); 457 } else if (bs.targetType != TargetType.none) { 458 BuildSettingsTemplate lib_settings; 459 lib_settings.targetType = bs.targetType == TargetType.autodetect ? TargetType.library : bs.targetType; 460 461 if (bs.targetType == TargetType.autodetect) { 462 if (app_main_file.length) { 463 lib_settings.excludedSourceFiles[""] ~= app_main_file; 464 465 BuildSettingsTemplate app_settings; 466 app_settings.targetType = TargetType.executable; 467 app_settings.mainSourceFile = app_main_file; 468 m_info.configurations ~= ConfigurationInfo("application", app_settings); 469 } 470 } 471 472 m_info.configurations ~= ConfigurationInfo("library", lib_settings); 473 } 474 } 475 } 476 477 private void simpleLint() const { 478 if (m_parentPackage) { 479 if (m_parentPackage.path != path) { 480 if (info.license.length && info.license != m_parentPackage.info.license) 481 logWarn("License in subpackage %s is different than it's parent package, this is discouraged.", name); 482 } 483 } 484 if (name.empty) logWarn("The package in %s has no name.", path); 485 } 486 487 private static RawPackage rawPackageFromFile(PathAndFormat file, bool silent_fail = false) { 488 if( silent_fail && !existsFile(file.path) ) return null; 489 490 string text; 491 492 { 493 auto f = openFile(file.path.toNativeString(), FileMode.Read); 494 scope(exit) f.close(); 495 text = stripUTF8Bom(cast(string)f.readAll()); 496 } 497 498 final switch(file.format) { 499 case PackageFormat.json: 500 return new JsonPackage(parseJsonString(text, file.path.toNativeString())); 501 case PackageFormat.sdl: 502 if(silent_fail) return null; throw new Exception("SDL not implemented"); 503 } 504 } 505 506 static abstract class RawPackage 507 { 508 string package_name; // Should already be lower case 509 string version_; 510 abstract void parseInto(ref PackageRecipe package_, string parent_name); 511 } 512 private static class JsonPackage : RawPackage 513 { 514 Json json; 515 this(Json json) { 516 this.json = json; 517 518 string nameLower; 519 if(json.type == Json.Type..string) { 520 nameLower = json.get!string.toLower(); 521 this.json = nameLower; 522 } else { 523 nameLower = json.name.get!string.toLower(); 524 this.json.name = nameLower; 525 this.package_name = nameLower; 526 527 Json versionJson = json["version"]; 528 this.version_ = (versionJson.type == Json.Type.undefined) ? null : versionJson.get!string; 529 } 530 531 this.package_name = nameLower; 532 } 533 override void parseInto(ref PackageRecipe recipe, string parent_name) 534 { 535 recipe.parseJson(json, parent_name); 536 } 537 } 538 private static class SdlPackage : RawPackage 539 { 540 override void parseInto(ref PackageRecipe package_, string parent_name) 541 { 542 throw new Exception("SDL packages not implemented yet"); 543 } 544 } 545 } 546 547 548 private string determineVersionFromSCM(Path path) 549 { 550 import std.process; 551 import dub.semver; 552 553 auto git_dir = path ~ ".git"; 554 if (!existsFile(git_dir) || !isDir(git_dir.toNativeString)) return null; 555 auto git_dir_param = "--git-dir=" ~ git_dir.toNativeString(); 556 557 static string exec(scope string[] params...) { 558 auto ret = executeShell(escapeShellCommand(params)); 559 if (ret.status == 0) return ret.output.strip; 560 logDebug("'%s' failed with exit code %s: %s", params.join(" "), ret.status, ret.output.strip); 561 return null; 562 } 563 564 if (auto tag = exec("git", git_dir_param, "describe", "--long", "--tags")) { 565 auto parts = tag.split("-"); 566 auto commit = parts[$-1]; 567 auto num = parts[$-2].to!int; 568 tag = parts[0 .. $-2].join("-"); 569 if (tag.startsWith("v") && isValidVersion(tag[1 .. $])) { 570 if (num == 0) return tag[1 .. $]; 571 else if (tag.canFind("+")) return format("%s.commit.%s.%s", tag[1 .. $], num, commit); 572 else return format("%s+commit.%s.%s", tag[1 .. $], num, commit); 573 } 574 } 575 576 if (auto branch = exec("git", git_dir_param, "rev-parse", "--abbrev-ref", "HEAD")) { 577 if (branch != "HEAD") return "~" ~ branch; 578 } 579 580 return null; 581 }