1 module dub.packagesuppliers.registry;
2 
3 import dub.dependency;
4 import dub.packagesuppliers.packagesupplier;
5 
6 package enum PackagesPath = "packages";
7 
8 /**
9 	Online registry based package supplier.
10 
11 	This package supplier connects to an online registry (e.g.
12 	$(LINK https://code.dlang.org/)) to search for available packages.
13 */
14 class RegistryPackageSupplier : PackageSupplier {
15 	import dub.internal.utils : retryDownload, HTTPStatusException;
16 	import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson;
17 	import dub.internal.vibecompat.inet.url : URL;
18 	import dub.internal.logging;
19 
20 	import std.uri : encodeComponent;
21 	import std.datetime : Clock, Duration, hours, SysTime, UTC;
22 	private {
23 		URL m_registryUrl;
24 		struct CacheEntry { Json data; SysTime cacheTime; }
25 		CacheEntry[PackageName] m_metadataCache;
26 		Duration m_maxCacheTime;
27 	}
28 
29  	this(URL registry)
30 	{
31 		m_registryUrl = registry;
32 		m_maxCacheTime = 24.hours();
33 	}
34 
35 	override @property string description() { return "registry at "~m_registryUrl.toString(); }
36 
37 	override Version[] getVersions(in PackageName name)
38 	{
39 		import std.algorithm.sorting : sort;
40 		auto md = getMetadata(name);
41 		if (md.type == Json.Type.null_)
42 			return null;
43 		Version[] ret;
44 		foreach (json; md["versions"]) {
45 			auto cur = Version(cast(string)json["version"]);
46 			ret ~= cur;
47 		}
48 		ret.sort();
49 		return ret;
50 	}
51 
52 	auto genPackageDownloadUrl(in PackageName name, in VersionRange dep, bool pre_release)
53 	{
54 		import std.array : replace;
55 		import std.format : format;
56 		import std.typecons : Nullable;
57 		auto md = getMetadata(name);
58 		Json best = getBestPackage(md, name, dep, pre_release);
59 		Nullable!URL ret;
60 		if (best.type != Json.Type.null_)
61 		{
62 			auto vers = best["version"].get!string;
63 			ret = m_registryUrl ~ NativePath(
64 				"%s/%s/%s.zip".format(PackagesPath, name.main, vers));
65 		}
66 		return ret;
67 	}
68 
69 	override ubyte[] fetchPackage(in PackageName name,
70 		in VersionRange dep, bool pre_release)
71 	{
72 		import std.format : format;
73 
74 		auto url = genPackageDownloadUrl(name, dep, pre_release);
75 		if(url.isNull) return null;
76 		try {
77 			return retryDownload(url.get);
78 		}
79 		catch(HTTPStatusException e) {
80 			if (e.status == 404) throw e;
81 			else logDebug("Failed to download package %s from %s", name.main, url);
82 		}
83 		catch(Exception e) {
84 			logDebug("Failed to download package %s from %s", name.main, url);
85 		}
86 		throw new Exception("Failed to download package %s from %s".format(name.main, url));
87 	}
88 
89 	override Json fetchPackageRecipe(in PackageName name, in VersionRange dep,
90 		bool pre_release)
91 	{
92 		auto md = getMetadata(name);
93 		return getBestPackage(md, name, dep, pre_release);
94 	}
95 
96 	private Json getMetadata(in PackageName name)
97 	{
98 		auto now = Clock.currTime(UTC());
99 		if (auto pentry = name.main in m_metadataCache) {
100 			if (pentry.cacheTime + m_maxCacheTime > now)
101 				return pentry.data;
102 			m_metadataCache.remove(name.main);
103 		}
104 
105 		auto url = m_registryUrl ~ NativePath("api/packages/infos");
106 
107 		url.queryString = "packages=" ~
108 			encodeComponent(`["` ~ name.main.toString() ~ `"]`) ~
109 			"&include_dependencies=true&minimize=true";
110 
111 		logDebug("Downloading metadata for %s", name.main);
112 		string jsonData;
113 
114 		jsonData = cast(string)retryDownload(url);
115 
116 		Json json = parseJsonString(jsonData, url.toString());
117 		foreach (pkg, info; json.get!(Json[string]))
118 		{
119 			logDebug("adding %s to metadata cache", pkg);
120 			m_metadataCache[PackageName(pkg)] = CacheEntry(info, now);
121 		}
122 		return json[name.main.toString()];
123 	}
124 
125 	SearchResult[] searchPackages(string query) {
126 		import std.array : array;
127 		import std.algorithm.iteration : map;
128 		import std.uri : encodeComponent;
129 		auto url = m_registryUrl;
130 		url.localURI = "/api/packages/search?q="~encodeComponent(query);
131 		string data;
132 		data = cast(string)retryDownload(url);
133 		return data.parseJson.opt!(Json[])
134 			.map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string))
135 			.array;
136 	}
137 }