1 module dub.packagesuppliers.registry; 2 3 import dub.packagesuppliers.packagesupplier; 4 5 package enum PackagesPath = "packages"; 6 7 /** 8 Online registry based package supplier. 9 10 This package supplier connects to an online registry (e.g. 11 $(LINK https://code.dlang.org/)) to search for available packages. 12 */ 13 class RegistryPackageSupplier : PackageSupplier { 14 import dub.internal.utils : download, retryDownload, HTTPStatusException; 15 import dub.internal.vibecompat.core.log; 16 import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson; 17 import dub.internal.vibecompat.inet.url : URL; 18 19 import std.datetime : Clock, Duration, hours, SysTime, UTC; 20 private { 21 URL m_registryUrl; 22 struct CacheEntry { Json data; SysTime cacheTime; } 23 CacheEntry[string] m_metadataCache; 24 Duration m_maxCacheTime; 25 } 26 27 this(URL registry) 28 { 29 m_registryUrl = registry; 30 m_maxCacheTime = 24.hours(); 31 } 32 33 override @property string description() { return "registry at "~m_registryUrl.toString(); } 34 35 Version[] getVersions(string package_id) 36 { 37 import std.algorithm.sorting : sort; 38 auto md = getMetadata(package_id); 39 if (md.type == Json.Type.null_) 40 return null; 41 Version[] ret; 42 foreach (json; md["versions"]) { 43 auto cur = Version(cast(string)json["version"]); 44 ret ~= cur; 45 } 46 ret.sort(); 47 return ret; 48 } 49 50 void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release) 51 { 52 import std.array : replace; 53 import std.format : format; 54 auto md = getMetadata(packageId); 55 Json best = getBestPackage(md, packageId, dep, pre_release); 56 if (best.type == Json.Type.null_) 57 return; 58 auto vers = best["version"].get!string; 59 auto url = m_registryUrl ~ NativePath(PackagesPath~"/"~packageId~"/"~vers~".zip"); 60 try { 61 retryDownload(url, path); 62 return; 63 } 64 catch(HTTPStatusException e) { 65 if (e.status == 404) throw e; 66 else logDebug("Failed to download package %s from %s", packageId, url); 67 } 68 catch(Exception e) { 69 logDebug("Failed to download package %s from %s", packageId, url); 70 } 71 throw new Exception("Failed to download package %s from %s".format(packageId, url)); 72 } 73 74 Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release) 75 { 76 auto md = getMetadata(packageId); 77 return getBestPackage(md, packageId, dep, pre_release); 78 } 79 80 private Json getMetadata(string packageId) 81 { 82 auto now = Clock.currTime(UTC()); 83 if (auto pentry = packageId in m_metadataCache) { 84 if (pentry.cacheTime + m_maxCacheTime > now) 85 return pentry.data; 86 m_metadataCache.remove(packageId); 87 } 88 89 auto url = m_registryUrl ~ NativePath("api/packages/infos?packages=[\"" ~ 90 packageId ~ "\"]&include_dependencies=true&minimize=true"); 91 92 logDebug("Downloading metadata for %s", packageId); 93 string jsonData; 94 95 jsonData = cast(string)retryDownload(url); 96 97 Json json = parseJsonString(jsonData, url.toString()); 98 foreach (pkg, info; json.get!(Json[string])) 99 { 100 logDebug("adding %s to metadata cache", pkg); 101 m_metadataCache[pkg] = CacheEntry(info, now); 102 } 103 return json[packageId]; 104 } 105 106 SearchResult[] searchPackages(string query) { 107 import std.array : array; 108 import std.algorithm.iteration : map; 109 import std.uri : encodeComponent; 110 auto url = m_registryUrl; 111 url.localURI = "/api/packages/search?q="~encodeComponent(query); 112 string data; 113 data = cast(string)retryDownload(url); 114 return data.parseJson.opt!(Json[]) 115 .map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string)) 116 .array; 117 } 118 } 119