1 module dub.packagesuppliers.registry; 2 3 import dub.dependency; 4 import dub.packagesuppliers.packagesupplier; 5 6 package enum PackagesPath = "packages"; 7 8 /** 9 Online registry based package supplier. 10 11 This package supplier connects to an online registry (e.g. 12 $(LINK https://code.dlang.org/)) to search for available packages. 13 */ 14 class RegistryPackageSupplier : PackageSupplier { 15 import dub.internal.utils : retryDownload, HTTPStatusException; 16 import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson; 17 import dub.internal.vibecompat.inet.path : InetPath; 18 import dub.internal.vibecompat.inet.url : URL; 19 import dub.internal.logging; 20 21 import std.uri : encodeComponent; 22 import std.datetime : Clock, Duration, hours, SysTime, UTC; 23 private { 24 URL m_registryUrl; 25 struct CacheEntry { Json data; SysTime cacheTime; } 26 CacheEntry[PackageName] m_metadataCache; 27 Duration m_maxCacheTime; 28 } 29 30 this(URL registry) 31 { 32 m_registryUrl = registry; 33 m_maxCacheTime = 24.hours(); 34 } 35 36 override @property string description() { return "registry at "~m_registryUrl.toString(); } 37 38 override Version[] getVersions(in PackageName name) 39 { 40 import std.algorithm.sorting : sort; 41 auto md = getMetadata(name); 42 if (md.type == Json.Type.null_) 43 return null; 44 Version[] ret; 45 foreach (json; md["versions"]) { 46 auto cur = Version(cast(string)json["version"]); 47 ret ~= cur; 48 } 49 ret.sort(); 50 return ret; 51 } 52 53 auto genPackageDownloadUrl(in PackageName name, in VersionRange dep, bool pre_release) 54 { 55 import std.array : replace; 56 import std.format : format; 57 import std.typecons : Nullable; 58 auto md = getMetadata(name); 59 Json best = getBestPackage(md, name, dep, pre_release); 60 Nullable!URL ret; 61 if (best.type != Json.Type.null_) 62 { 63 auto vers = best["version"].get!string; 64 ret = m_registryUrl ~ InetPath( 65 "%s/%s/%s.zip".format(PackagesPath, name.main, vers)); 66 } 67 return ret; 68 } 69 70 override ubyte[] fetchPackage(in PackageName name, 71 in VersionRange dep, bool pre_release) 72 { 73 import std.format : format; 74 75 auto url = genPackageDownloadUrl(name, dep, pre_release); 76 if(url.isNull) return null; 77 try { 78 return retryDownload(url.get); 79 } 80 catch(HTTPStatusException e) { 81 if (e.status == 404) throw e; 82 else logDebug("Failed to download package %s from %s", name.main, url); 83 } 84 catch(Exception e) { 85 logDebug("Failed to download package %s from %s", name.main, url); 86 } 87 throw new Exception("Failed to download package %s from %s".format(name.main, url)); 88 } 89 90 override Json fetchPackageRecipe(in PackageName name, in VersionRange dep, 91 bool pre_release) 92 { 93 auto md = getMetadata(name); 94 return getBestPackage(md, name, dep, pre_release); 95 } 96 97 private Json getMetadata(in PackageName name) 98 { 99 auto now = Clock.currTime(UTC()); 100 if (auto pentry = name.main in m_metadataCache) { 101 if (pentry.cacheTime + m_maxCacheTime > now) 102 return pentry.data; 103 m_metadataCache.remove(name.main); 104 } 105 106 auto url = m_registryUrl ~ InetPath("api/packages/infos"); 107 108 url.queryString = "packages=" ~ 109 encodeComponent(`["` ~ name.main.toString() ~ `"]`) ~ 110 "&include_dependencies=true&minimize=true"; 111 112 logDebug("Downloading metadata for %s", name.main); 113 string jsonData; 114 115 jsonData = cast(string)retryDownload(url); 116 117 Json json = parseJsonString(jsonData, url.toString()); 118 foreach (pkg, info; json.get!(Json[string])) 119 { 120 logDebug("adding %s to metadata cache", pkg); 121 m_metadataCache[PackageName(pkg)] = CacheEntry(info, now); 122 } 123 return json[name.main.toString()]; 124 } 125 126 SearchResult[] searchPackages(string query) { 127 import std.array : array; 128 import std.algorithm.iteration : map; 129 import std.uri : encodeComponent; 130 auto url = m_registryUrl; 131 url.localURI = "/api/packages/search?q="~encodeComponent(query); 132 string data; 133 data = cast(string)retryDownload(url); 134 return data.parseJson.opt!(Json[]) 135 .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string)) 136 .array; 137 } 138 }