1 module dub.packagesuppliers.registry;
2 
3 import dub.packagesuppliers.packagesupplier;
4 
5 package enum PackagesPath = "packages";
6 
7 /**
8 	Online registry based package supplier.
9 
10 	This package supplier connects to an online registry (e.g.
11 	$(LINK https://code.dlang.org/)) to search for available packages.
12 */
13 class RegistryPackageSupplier : PackageSupplier {
14 	import dub.internal.utils : download, retryDownload, HTTPStatusException;
15 	import dub.internal.vibecompat.data.json : parseJson, parseJsonString, serializeToJson;
16 	import dub.internal.vibecompat.inet.url : URL;
17 	import dub.internal.logging;
18 
19 	import std.uri : encodeComponent;
20 	import std.datetime : Clock, Duration, hours, SysTime, UTC;
21 	private {
22 		URL m_registryUrl;
23 		struct CacheEntry { Json data; SysTime cacheTime; }
24 		CacheEntry[string] m_metadataCache;
25 		Duration m_maxCacheTime;
26 	}
27 
28  	this(URL registry)
29 	{
30 		m_registryUrl = registry;
31 		m_maxCacheTime = 24.hours();
32 	}
33 
34 	override @property string description() { return "registry at "~m_registryUrl.toString(); }
35 
36 	Version[] getVersions(string package_id)
37 	{
38 		import std.algorithm.sorting : sort;
39 		auto md = getMetadata(package_id);
40 		if (md.type == Json.Type.null_)
41 			return null;
42 		Version[] ret;
43 		foreach (json; md["versions"]) {
44 			auto cur = Version(cast(string)json["version"]);
45 			ret ~= cur;
46 		}
47 		ret.sort();
48 		return ret;
49 	}
50 
51 	auto genPackageDownloadUrl(string packageId, Dependency dep, bool pre_release)
52 	{
53 		import std.array : replace;
54 		import std.format : format;
55 		import std.typecons : Nullable;
56 		auto md = getMetadata(packageId);
57 		Json best = getBestPackage(md, packageId, dep, pre_release);
58 		Nullable!URL ret;
59 		if (best.type != Json.Type.null_)
60 		{
61 			auto vers = best["version"].get!string;
62 			ret = m_registryUrl ~ NativePath(PackagesPath~"/"~packageId~"/"~vers~".zip");
63 		}
64 		return ret;
65 	}
66 
67 	void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release)
68 	{
69 		import std.format : format;
70 		auto url = genPackageDownloadUrl(packageId, dep, pre_release);
71 		if(url.isNull)
72 			return;
73 		try {
74 			retryDownload(url.get, path);
75 			return;
76 		}
77 		catch(HTTPStatusException e) {
78 			if (e.status == 404) throw e;
79 			else logDebug("Failed to download package %s from %s", packageId, url);
80 		}
81 		catch(Exception e) {
82 			logDebug("Failed to download package %s from %s", packageId, url);
83 		}
84 		throw new Exception("Failed to download package %s from %s".format(packageId, url));
85 	}
86 
87 	Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release)
88 	{
89 		auto md = getMetadata(packageId);
90 		return getBestPackage(md, packageId, dep, pre_release);
91 	}
92 
93 	private Json getMetadata(string packageId)
94 	{
95 		auto now = Clock.currTime(UTC());
96 		if (auto pentry = packageId in m_metadataCache) {
97 			if (pentry.cacheTime + m_maxCacheTime > now)
98 				return pentry.data;
99 			m_metadataCache.remove(packageId);
100 		}
101 
102 		auto url = m_registryUrl ~ NativePath("api/packages/infos");
103 
104 		url.queryString = "packages=" ~
105 				encodeComponent(`["` ~ packageId ~ `"]`) ~ "&include_dependencies=true&minimize=true";
106 
107 		logDebug("Downloading metadata for %s", packageId);
108 		string jsonData;
109 
110 		jsonData = cast(string)retryDownload(url);
111 
112 		Json json = parseJsonString(jsonData, url.toString());
113 		foreach (pkg, info; json.get!(Json[string]))
114 		{
115 			logDebug("adding %s to metadata cache", pkg);
116 			m_metadataCache[pkg] = CacheEntry(info, now);
117 		}
118 		return json[packageId];
119 	}
120 
121 	SearchResult[] searchPackages(string query) {
122 		import std.array : array;
123 		import std.algorithm.iteration : map;
124 		import std.uri : encodeComponent;
125 		auto url = m_registryUrl;
126 		url.localURI = "/api/packages/search?q="~encodeComponent(query);
127 		string data;
128 		data = cast(string)retryDownload(url);
129 		return data.parseJson.opt!(Json[])
130 			.map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string))
131 			.array;
132 	}
133 }