1 module dub.packagesuppliers.registry; 2 3 import dub.packagesuppliers.packagesupplier; 4 5 package enum PackagesPath = "packages"; 6 7 /** 8 Online registry based package supplier. 9 10 This package supplier connects to an online registry (e.g. 11 $(LINK https://code.dlang.org/)) to search for available packages. 12 */ 13 class RegistryPackageSupplier : PackageSupplier { 14 import dub.internal.utils : download, retryDownload, HTTPStatusException; 15 import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson; 16 import dub.internal.vibecompat.inet.url : URL; 17 import dub.internal.logging; 18 19 import std.uri : encodeComponent; 20 import std.datetime : Clock, Duration, hours, SysTime, UTC; 21 private { 22 URL m_registryUrl; 23 struct CacheEntry { Json data; SysTime cacheTime; } 24 CacheEntry[string] m_metadataCache; 25 Duration m_maxCacheTime; 26 } 27 28 this(URL registry) 29 { 30 m_registryUrl = registry; 31 m_maxCacheTime = 24.hours(); 32 } 33 34 override @property string description() { return "registry at "~m_registryUrl.toString(); } 35 36 Version[] getVersions(string package_id) 37 { 38 import std.algorithm.sorting : sort; 39 auto md = getMetadata(package_id); 40 if (md.type == Json.Type.null_) 41 return null; 42 Version[] ret; 43 foreach (json; md["versions"]) { 44 auto cur = Version(cast(string)json["version"]); 45 ret ~= cur; 46 } 47 ret.sort(); 48 return ret; 49 } 50 51 auto genPackageDownloadUrl(string packageId, Dependency dep, bool pre_release) 52 { 53 import std.array : replace; 54 import std.format : format; 55 import std.typecons : Nullable; 56 auto md = getMetadata(packageId); 57 Json best = getBestPackage(md, packageId, dep, pre_release); 58 Nullable!URL ret; 59 if (best.type != Json.Type.null_) 60 { 61 auto vers = best["version"].get!string; 62 ret = m_registryUrl ~ NativePath(PackagesPath~"/"~packageId~"/"~vers~".zip"); 63 } 64 return ret; 65 } 66 67 void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release) 68 { 69 import std.format : format; 70 auto url = genPackageDownloadUrl(packageId, dep, pre_release); 71 if(url.isNull) 72 return; 73 try { 74 retryDownload(url.get, path); 75 return; 76 } 77 catch(HTTPStatusException e) { 78 if (e.status == 404) throw e; 79 else logDebug("Failed to download package %s from %s", packageId, url); 80 } 81 catch(Exception e) { 82 logDebug("Failed to download package %s from %s", packageId, url); 83 } 84 throw new Exception("Failed to download package %s from %s".format(packageId, url)); 85 } 86 87 Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release) 88 { 89 auto md = getMetadata(packageId); 90 return getBestPackage(md, packageId, dep, pre_release); 91 } 92 93 private Json getMetadata(string packageId) 94 { 95 auto now = Clock.currTime(UTC()); 96 if (auto pentry = packageId in m_metadataCache) { 97 if (pentry.cacheTime + m_maxCacheTime > now) 98 return pentry.data; 99 m_metadataCache.remove(packageId); 100 } 101 102 auto url = m_registryUrl ~ NativePath("api/packages/infos"); 103 104 url.queryString = "packages=" ~ 105 encodeComponent(`["` ~ packageId ~ `"]`) ~ "&include_dependencies=true&minimize=true"; 106 107 logDebug("Downloading metadata for %s", packageId); 108 string jsonData; 109 110 jsonData = cast(string)retryDownload(url); 111 112 Json json = parseJsonString(jsonData, url.toString()); 113 foreach (pkg, info; json.get!(Json[string])) 114 { 115 logDebug("adding %s to metadata cache", pkg); 116 m_metadataCache[pkg] = CacheEntry(info, now); 117 } 118 return json[packageId]; 119 } 120 121 SearchResult[] searchPackages(string query) { 122 import std.array : array; 123 import std.algorithm.iteration : map; 124 import std.uri : encodeComponent; 125 auto url = m_registryUrl; 126 url.localURI = "/api/packages/search?q="~encodeComponent(query); 127 string data; 128 data = cast(string)retryDownload(url); 129 return data.parseJson.opt!(Json[]) 130 .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string)) 131 .array; 132 } 133 }