1 module dub.packagesuppliers.registry; 2 3 import dub.packagesuppliers.packagesupplier; 4 5 package enum PackagesPath = "packages"; 6 7 /** 8 Online registry based package supplier. 9 10 This package supplier connects to an online registry (e.g. 11 $(LINK https://code.dlang.org/)) to search for available packages. 12 */ 13 class RegistryPackageSupplier : PackageSupplier { 14 import dub.internal.utils : download, retryDownload, HTTPStatusException; 15 import dub.internal.vibecompat.core.log; 16 import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson; 17 import dub.internal.vibecompat.inet.url : URL; 18 19 import std.datetime : Clock, Duration, hours, SysTime, UTC; 20 private { 21 URL m_registryUrl; 22 struct CacheEntry { Json data; SysTime cacheTime; } 23 CacheEntry[string] m_metadataCache; 24 Duration m_maxCacheTime; 25 } 26 27 this(URL registry) 28 { 29 m_registryUrl = registry; 30 m_maxCacheTime = 24.hours(); 31 } 32 33 override @property string description() { return "registry at "~m_registryUrl.toString(); } 34 35 Version[] getVersions(string package_id) 36 { 37 import std.algorithm.sorting : sort; 38 auto md = getMetadata(package_id); 39 if (md.type == Json.Type.null_) 40 return null; 41 Version[] ret; 42 foreach (json; md["versions"]) { 43 auto cur = Version(cast(string)json["version"]); 44 ret ~= cur; 45 } 46 ret.sort(); 47 return ret; 48 } 49 50 void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release) 51 { 52 import std.array : replace; 53 import std.format : format; 54 auto md = getMetadata(packageId); 55 Json best = getBestPackage(md, packageId, dep, pre_release); 56 if (best.type == Json.Type.null_) 57 return; 58 auto vers = best["version"].get!string; 59 auto url = m_registryUrl ~ NativePath(PackagesPath~"/"~packageId~"/"~vers~".zip"); 60 try { 61 retryDownload(url, path); 62 return; 63 } 64 catch(HTTPStatusException e) { 65 if (e.status == 404) throw e; 66 else logDebug("Failed to download package %s from %s", packageId, url); 67 } 68 catch(Exception e) { 69 logDebug("Failed to download package %s from %s", packageId, url); 70 } 71 throw new Exception("Failed to download package %s from %s".format(packageId, url)); 72 } 73 74 Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release) 75 { 76 auto md = getMetadata(packageId); 77 return getBestPackage(md, packageId, dep, pre_release); 78 } 79 80 private Json getMetadata(string packageId) 81 { 82 auto now = Clock.currTime(UTC()); 83 if (auto pentry = packageId in m_metadataCache) { 84 if (pentry.cacheTime + m_maxCacheTime > now) 85 return pentry.data; 86 m_metadataCache.remove(packageId); 87 } 88 89 auto url = m_registryUrl ~ NativePath(PackagesPath ~ "/" ~ packageId ~ ".json"); 90 91 logDebug("Downloading metadata for %s", packageId); 92 string jsonData; 93 94 try 95 jsonData = cast(string)retryDownload(url); 96 catch(HTTPStatusException e) { 97 if (e.status == 404) { 98 logDebug("Package %s not found at %s (404): %s", packageId, description, e.msg); 99 return Json(null); 100 } 101 else throw e; 102 } 103 104 Json json = parseJsonString(jsonData, url.toString()); 105 // strip readme data (to save size and time) 106 foreach (ref v; json["versions"]) 107 v.remove("readme"); 108 m_metadataCache[packageId] = CacheEntry(json, now); 109 return json; 110 } 111 112 SearchResult[] searchPackages(string query) { 113 import std.array : array; 114 import std.algorithm.iteration : map; 115 import std.uri : encodeComponent; 116 auto url = m_registryUrl; 117 url.localURI = "/api/packages/search?q="~encodeComponent(query); 118 string data; 119 data = cast(string)retryDownload(url); 120 return data.parseJson.opt!(Json[]) 121 .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string)) 122 .array; 123 } 124 } 125