1 module dub.packagesuppliers.registry;
2 
3 import dub.dependency;
4 import dub.packagesuppliers.packagesupplier;
5 
6 package enum PackagesPath = "packages";
7 
8 /**
9 	Online registry based package supplier.
10 
11 	This package supplier connects to an online registry (e.g.
12 	$(LINK https://code.dlang.org/)) to search for available packages.
13 */
14 class RegistryPackageSupplier : PackageSupplier {
15 	import dub.internal.utils : retryDownload, HTTPStatusException;
16 	import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson;
17 	import dub.internal.vibecompat.inet.path : InetPath;
18 	import dub.internal.vibecompat.inet.url : URL;
19 	import dub.internal.logging;
20 
21 	import std.uri : encodeComponent;
22 	import std.datetime : Clock, Duration, hours, SysTime, UTC;
23 	private {
24 		URL m_registryUrl;
25 		struct CacheEntry { Json data; SysTime cacheTime; }
26 		CacheEntry[PackageName] m_metadataCache;
27 		Duration m_maxCacheTime;
28 	}
29 
30  	this(URL registry)
31 	{
32 		m_registryUrl = registry;
33 		m_maxCacheTime = 24.hours();
34 	}
35 
36 	override @property string description() { return "registry at "~m_registryUrl.toString(); }
37 
38 	override Version[] getVersions(in PackageName name)
39 	{
40 		import std.algorithm.sorting : sort;
41 		auto md = getMetadata(name);
42 		if (md.type == Json.Type.null_)
43 			return null;
44 		Version[] ret;
45 		foreach (json; md["versions"]) {
46 			auto cur = Version(cast(string)json["version"]);
47 			ret ~= cur;
48 		}
49 		ret.sort();
50 		return ret;
51 	}
52 
53 	auto genPackageDownloadUrl(in PackageName name, in VersionRange dep, bool pre_release)
54 	{
55 		import std.array : replace;
56 		import std.format : format;
57 		import std.typecons : Nullable;
58 		auto md = getMetadata(name);
59 		Json best = getBestPackage(md, name, dep, pre_release);
60 		Nullable!URL ret;
61 		if (best.type != Json.Type.null_)
62 		{
63 			auto vers = best["version"].get!string;
64 			ret = m_registryUrl ~ InetPath(
65 				"%s/%s/%s.zip".format(PackagesPath, name.main, vers));
66 		}
67 		return ret;
68 	}
69 
70 	override ubyte[] fetchPackage(in PackageName name,
71 		in VersionRange dep, bool pre_release)
72 	{
73 		import std.format : format;
74 
75 		auto url = genPackageDownloadUrl(name, dep, pre_release);
76 		if(url.isNull) return null;
77 		try {
78 			return retryDownload(url.get);
79 		}
80 		catch(HTTPStatusException e) {
81 			if (e.status == 404) throw e;
82 			else logDebug("Failed to download package %s from %s", name.main, url);
83 		}
84 		catch(Exception e) {
85 			logDebug("Failed to download package %s from %s", name.main, url);
86 		}
87 		throw new Exception("Failed to download package %s from %s".format(name.main, url));
88 	}
89 
90 	override Json fetchPackageRecipe(in PackageName name, in VersionRange dep,
91 		bool pre_release)
92 	{
93 		auto md = getMetadata(name);
94 		return getBestPackage(md, name, dep, pre_release);
95 	}
96 
97 	private Json getMetadata(in PackageName name)
98 	{
99 		auto now = Clock.currTime(UTC());
100 		if (auto pentry = name.main in m_metadataCache) {
101 			if (pentry.cacheTime + m_maxCacheTime > now)
102 				return pentry.data;
103 			m_metadataCache.remove(name.main);
104 		}
105 
106 		auto url = m_registryUrl ~ InetPath("api/packages/infos");
107 
108 		url.queryString = "packages=" ~
109 			encodeComponent(`["` ~ name.main.toString() ~ `"]`) ~
110 			"&include_dependencies=true&minimize=true";
111 
112 		logDebug("Downloading metadata for %s", name.main);
113 		string jsonData;
114 
115 		jsonData = cast(string)retryDownload(url);
116 
117 		Json json = parseJsonString(jsonData, url.toString());
118 		foreach (pkg, info; json.get!(Json[string]))
119 		{
120 			logDebug("adding %s to metadata cache", pkg);
121 			m_metadataCache[PackageName(pkg)] = CacheEntry(info, now);
122 		}
123 		return json[name.main.toString()];
124 	}
125 
126 	SearchResult[] searchPackages(string query) {
127 		import std.array : array;
128 		import std.algorithm.iteration : map;
129 		import std.uri : encodeComponent;
130 		auto url = m_registryUrl;
131 		url.localURI = "/api/packages/search?q="~encodeComponent(query);
132 		string data;
133 		data = cast(string)retryDownload(url);
134 		return data.parseJson.opt!(Json[])
135 			.map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string))
136 			.array;
137 	}
138 }