1 module dub.packagesuppliers.registry; 2 3 import dub.dependency; 4 import dub.packagesuppliers.packagesupplier; 5 6 package enum PackagesPath = "packages"; 7 8 /** 9 Online registry based package supplier. 10 11 This package supplier connects to an online registry (e.g. 12 $(LINK https://code.dlang.org/)) to search for available packages. 13 */ 14 class RegistryPackageSupplier : PackageSupplier { 15 import dub.internal.utils : retryDownload, HTTPStatusException; 16 import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson; 17 import dub.internal.vibecompat.inet.url : URL; 18 import dub.internal.logging; 19 20 import std.uri : encodeComponent; 21 import std.datetime : Clock, Duration, hours, SysTime, UTC; 22 private { 23 URL m_registryUrl; 24 struct CacheEntry { Json data; SysTime cacheTime; } 25 CacheEntry[PackageName] m_metadataCache; 26 Duration m_maxCacheTime; 27 } 28 29 this(URL registry) 30 { 31 m_registryUrl = registry; 32 m_maxCacheTime = 24.hours(); 33 } 34 35 override @property string description() { return "registry at "~m_registryUrl.toString(); } 36 37 override Version[] getVersions(in PackageName name) 38 { 39 import std.algorithm.sorting : sort; 40 auto md = getMetadata(name); 41 if (md.type == Json.Type.null_) 42 return null; 43 Version[] ret; 44 foreach (json; md["versions"]) { 45 auto cur = Version(cast(string)json["version"]); 46 ret ~= cur; 47 } 48 ret.sort(); 49 return ret; 50 } 51 52 auto genPackageDownloadUrl(in PackageName name, in VersionRange dep, bool pre_release) 53 { 54 import std.array : replace; 55 import std.format : format; 56 import std.typecons : Nullable; 57 auto md = getMetadata(name); 58 Json best = getBestPackage(md, name, dep, pre_release); 59 Nullable!URL ret; 60 if (best.type != Json.Type.null_) 61 { 62 auto vers = best["version"].get!string; 63 ret = m_registryUrl ~ NativePath( 64 "%s/%s/%s.zip".format(PackagesPath, name.main, vers)); 65 } 66 return ret; 67 } 68 69 override ubyte[] fetchPackage(in PackageName name, 70 in VersionRange dep, bool pre_release) 71 { 72 import std.format : format; 73 74 auto url = genPackageDownloadUrl(name, dep, pre_release); 75 if(url.isNull) return null; 76 try { 77 return retryDownload(url.get); 78 } 79 catch(HTTPStatusException e) { 80 if (e.status == 404) throw e; 81 else logDebug("Failed to download package %s from %s", name.main, url); 82 } 83 catch(Exception e) { 84 logDebug("Failed to download package %s from %s", name.main, url); 85 } 86 throw new Exception("Failed to download package %s from %s".format(name.main, url)); 87 } 88 89 override Json fetchPackageRecipe(in PackageName name, in VersionRange dep, 90 bool pre_release) 91 { 92 auto md = getMetadata(name); 93 return getBestPackage(md, name, dep, pre_release); 94 } 95 96 private Json getMetadata(in PackageName name) 97 { 98 auto now = Clock.currTime(UTC()); 99 if (auto pentry = name.main in m_metadataCache) { 100 if (pentry.cacheTime + m_maxCacheTime > now) 101 return pentry.data; 102 m_metadataCache.remove(name.main); 103 } 104 105 auto url = m_registryUrl ~ NativePath("api/packages/infos"); 106 107 url.queryString = "packages=" ~ 108 encodeComponent(`["` ~ name.main.toString() ~ `"]`) ~ 109 "&include_dependencies=true&minimize=true"; 110 111 logDebug("Downloading metadata for %s", name.main); 112 string jsonData; 113 114 jsonData = cast(string)retryDownload(url); 115 116 Json json = parseJsonString(jsonData, url.toString()); 117 foreach (pkg, info; json.get!(Json[string])) 118 { 119 logDebug("adding %s to metadata cache", pkg); 120 m_metadataCache[PackageName(pkg)] = CacheEntry(info, now); 121 } 122 return json[name.main.toString()]; 123 } 124 125 SearchResult[] searchPackages(string query) { 126 import std.array : array; 127 import std.algorithm.iteration : map; 128 import std.uri : encodeComponent; 129 auto url = m_registryUrl; 130 url.localURI = "/api/packages/search?q="~encodeComponent(query); 131 string data; 132 data = cast(string)retryDownload(url); 133 return data.parseJson.opt!(Json[]) 134 .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string)) 135 .array; 136 } 137 }