1 /**
2 	Contains (remote) package supplier interface and implementations.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff, 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff
7 */
8 module dub.packagesupplier;
9 
10 import dub.dependency;
11 import dub.internal.utils;
12 import dub.internal.vibecompat.core.log;
13 import dub.internal.vibecompat.core.file;
14 import dub.internal.vibecompat.data.json;
15 import dub.internal.vibecompat.inet.url;
16 
17 import std.algorithm : filter, sort;
18 import std.array : array;
19 import std.conv;
20 import std.datetime;
21 import std.exception;
22 import std.file;
23 import std..string : format;
24 import std.typecons : AutoImplement;
25 import std.zip;
26 
27 // TODO: Could drop the "best package" behavior and let retrievePackage/
28 //       getPackageDescription take a Version instead of Dependency. But note
29 //       this means that two requests to the registry are necessary to retrieve
30 //       a package recipe instead of one (first get version list, then the
31 //       package recipe)
32 
33 /**
34 	Base interface for remote package suppliers.
35 
36 	Provides functionality necessary to query package versions, recipes and
37 	contents.
38 */
39 interface PackageSupplier {
40 	/// Represents a single package search result.
41 	static struct SearchResult { string name, description, version_; }
42 
43 	/// Returns a human-readable representation of the package supplier.
44 	@property string description();
45 
46 	/** Retrieves a list of all available versions(/branches) of a package.
47 
48 		Throws: Throws an exception if the package name is not known, or if
49 			an error occurred while retrieving the version list.
50 	*/
51 	Version[] getVersions(string package_id);
52 
53 	/** Downloads a package and stores it as a ZIP file.
54 
55 		Params:
56 			path = Absolute path of the target ZIP file
57 			package_id = Name of the package to retrieve
58 			dep: Version constraint to match against
59 			pre_release: If true, matches the latest pre-release version.
60 				Otherwise prefers stable versions.
61 	*/
62 	void fetchPackage(NativePath path, string package_id, Dependency dep, bool pre_release);
63 
64 	/** Retrieves only the recipe of a particular package.
65 
66 		Params:
67 			package_id = Name of the package of which to retrieve the recipe
68 			dep: Version constraint to match against
69 			pre_release: If true, matches the latest pre-release version.
70 				Otherwise prefers stable versions.
71 	*/
72 	Json fetchPackageRecipe(string package_id, Dependency dep, bool pre_release);
73 
74 	/** Searches for packages matching the given search query term.
75 
76 		Search queries are currently a simple list of words separated by
77 		white space. Results will get ordered from best match to worst.
78 	*/
79 	SearchResult[] searchPackages(string query);
80 }
81 
82 
83 /**
84 	File system based package supplier.
85 
86 	This package supplier searches a certain directory for files with names of
87 	the form "[package name]-[version].zip".
88 */
89 class FileSystemPackageSupplier : PackageSupplier {
90 	private {
91 		NativePath m_path;
92 	}
93 
94 	this(NativePath root) { m_path = root; }
95 
96 	override @property string description() { return "file repository at "~m_path.toNativeString(); }
97 
98 	Version[] getVersions(string package_id)
99 	{
100 		Version[] ret;
101 		foreach (DirEntry d; dirEntries(m_path.toNativeString(), package_id~"*", SpanMode.shallow)) {
102 			NativePath p = NativePath(d.name);
103 			logDebug("Entry: %s", p);
104 			enforce(to!string(p.head)[$-4..$] == ".zip");
105 			auto vers = p.head.toString()[package_id.length+1..$-4];
106 			logDebug("Version: %s", vers);
107 			ret ~= Version(vers);
108 		}
109 		ret.sort();
110 		return ret;
111 	}
112 
113 	void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release)
114 	{
115 		enforce(path.absolute);
116 		logInfo("Storing package '"~packageId~"', version requirements: %s", dep);
117 		auto filename = bestPackageFile(packageId, dep, pre_release);
118 		enforce(existsFile(filename));
119 		copyFile(filename, path);
120 	}
121 
122 	Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release)
123 	{
124 		auto filename = bestPackageFile(packageId, dep, pre_release);
125 		return jsonFromZip(filename, "dub.json");
126 	}
127 
128 	SearchResult[] searchPackages(string query)
129 	{
130 		// TODO!
131 		return null;
132 	}
133 
134 	private NativePath bestPackageFile(string packageId, Dependency dep, bool pre_release)
135 	{
136 		NativePath toPath(Version ver) {
137 			return m_path ~ (packageId ~ "-" ~ ver.toString() ~ ".zip");
138 		}
139 		auto versions = getVersions(packageId).filter!(v => dep.matches(v)).array;
140 		enforce(versions.length > 0, format("No package %s found matching %s", packageId, dep));
141 		foreach_reverse (ver; versions) {
142 			if (pre_release || !ver.isPreRelease)
143 				return toPath(ver);
144 		}
145 		return toPath(versions[$-1]);
146 	}
147 }
148 
149 
150 /**
151 	Online registry based package supplier.
152 
153 	This package supplier connects to an online registry (e.g.
154 	$(LINK https://code.dlang.org/)) to search for available packages.
155 */
156 class RegistryPackageSupplier : PackageSupplier {
157 	private {
158 		URL m_registryUrl;
159 		struct CacheEntry { Json data; SysTime cacheTime; }
160 		CacheEntry[string] m_metadataCache;
161 		Duration m_maxCacheTime;
162 	}
163 
164  	this(URL registry)
165 	{
166 		m_registryUrl = registry;
167 		m_maxCacheTime = 24.hours();
168 	}
169 
170 	override @property string description() { return "registry at "~m_registryUrl.toString(); }
171 
172 	Version[] getVersions(string package_id)
173 	{
174 		auto md = getMetadata(package_id);
175 		if (md.type == Json.Type.null_)
176 			return null;
177 		Version[] ret;
178 		foreach (json; md["versions"]) {
179 			auto cur = Version(cast(string)json["version"]);
180 			ret ~= cur;
181 		}
182 		ret.sort();
183 		return ret;
184 	}
185 
186 	void fetchPackage(NativePath path, string packageId, Dependency dep, bool pre_release)
187 	{
188 		import std.array : replace;
189 		Json best = getBestPackage(packageId, dep, pre_release);
190 		if (best.type == Json.Type.null_)
191 			return;
192 		auto vers = best["version"].get!string;
193 		auto url = m_registryUrl ~ NativePath(PackagesPath~"/"~packageId~"/"~vers~".zip");
194 		logDiagnostic("Downloading from '%s'", url);
195 		foreach(i; 0..3) {
196 			try{
197 				download(url, path);
198 				return;
199 			}
200 			catch(HTTPStatusException e) {
201 				if (e.status == 404) throw e;
202 				else {
203 					logDebug("Failed to download package %s from %s (Attempt %s of 3)", packageId, url, i + 1);
204 					continue;
205 				}
206 			}
207 		}
208 		throw new Exception("Failed to download package %s from %s".format(packageId, url));
209 	}
210 
211 	Json fetchPackageRecipe(string packageId, Dependency dep, bool pre_release)
212 	{
213 		return getBestPackage(packageId, dep, pre_release);
214 	}
215 
216 	private Json getMetadata(string packageId)
217 	{
218 		auto now = Clock.currTime(UTC());
219 		if (auto pentry = packageId in m_metadataCache) {
220 			if (pentry.cacheTime + m_maxCacheTime > now)
221 				return pentry.data;
222 			m_metadataCache.remove(packageId);
223 		}
224 
225 		auto url = m_registryUrl ~ NativePath(PackagesPath ~ "/" ~ packageId ~ ".json");
226 
227 		logDebug("Downloading metadata for %s", packageId);
228 		logDebug("Getting from %s", url);
229 
230 		string jsonData;
231 		foreach(i; 0..3) {
232 			try {
233 				jsonData = cast(string)download(url);
234 				break;
235 			}
236 			catch (HTTPStatusException e)
237 			{
238 				if (e.status == 404) {
239 					logDebug("Package %s not found at %s (404): %s", packageId, description, e.msg);
240 					return Json(null);
241 				}
242 				else {
243 					logDebug("Error getting metadata for package %s at %s (attempt %s of 3): %s", packageId, description, i + 1, e.msg);
244 					if (i == 2)
245 						throw e;
246 					continue;
247 				}
248 			}
249 		}
250 		Json json = parseJsonString(jsonData, url.toString());
251 		// strip readme data (to save size and time)
252 		foreach (ref v; json["versions"])
253 			v.remove("readme");
254 		m_metadataCache[packageId] = CacheEntry(json, now);
255 		return json;
256 	}
257 
258 	SearchResult[] searchPackages(string query) {
259 		import std.uri : encodeComponent;
260 		auto url = m_registryUrl;
261 		url.localURI = "/api/packages/search?q="~encodeComponent(query);
262 		string data;
263 		data = cast(string)download(url);
264 		import std.algorithm : map;
265 		return data.parseJson.opt!(Json[])
266 			.map!(j => SearchResult(j["name"].opt!string, j["description"].opt!string, j["version"].opt!string))
267 			.array;
268 	}
269 
270 	private Json getBestPackage(string packageId, Dependency dep, bool pre_release)
271 	{
272 		Json md = getMetadata(packageId);
273 		if (md.type == Json.Type.null_)
274 			return md;
275 		Json best = null;
276 		Version bestver;
277 		foreach (json; md["versions"]) {
278 			auto cur = Version(cast(string)json["version"]);
279 			if (!dep.matches(cur)) continue;
280 			if (best == null) best = json;
281 			else if (pre_release) {
282 				if (cur > bestver) best = json;
283 			} else if (bestver.isPreRelease) {
284 				if (!cur.isPreRelease || cur > bestver) best = json;
285 			} else if (!cur.isPreRelease && cur > bestver) best = json;
286 			bestver = Version(cast(string)best["version"]);
287 		}
288 		enforce(best != null, "No package candidate found for "~packageId~" "~dep.toString());
289 		return best;
290 	}
291 }
292 
293 package abstract class AbstractFallbackPackageSupplier : PackageSupplier
294 {
295 	protected PackageSupplier m_default;
296 	protected PackageSupplier[] m_fallbacks;
297 
298 	this(PackageSupplier default_, PackageSupplier[] fallbacks)
299 	{
300 		m_default = default_;
301 		m_fallbacks = fallbacks;
302 	}
303 
304 	override @property string description()
305 	{
306 		import std.algorithm : map;
307 		return format("%s (fallback %s)", m_default.description, m_fallbacks.map!(x => x.description));
308 	}
309 
310 	// Workaround https://issues.dlang.org/show_bug.cgi?id=2525
311 	abstract override Version[] getVersions(string package_id);
312 	abstract override void fetchPackage(NativePath path, string package_id, Dependency dep, bool pre_release);
313 	abstract override Json fetchPackageRecipe(string package_id, Dependency dep, bool pre_release);
314 	abstract override SearchResult[] searchPackages(string query);
315 }
316 
317 /**
318 	Combines two package suppliers and uses the second as fallback to handle failures.
319 
320 	Assumes that both registries serve the same packages (--mirror).
321 */
322 package alias FallbackPackageSupplier = AutoImplement!(AbstractFallbackPackageSupplier, fallback);
323 
324 private template fallback(T, alias func)
325 {
326 	enum fallback = q{
327 		import std.range : back, dropBackOne;
328 		import dub.internal.vibecompat.core.log : logDebug;
329 		scope (failure)
330 		{
331 			foreach (m_fallback; m_fallbacks.dropBackOne)
332 			{
333 				try
334 					return m_fallback.%1$s(args);
335 				catch(Exception)
336 					logDebug("Package supplier %s failed. Trying next fallback.", m_fallback);
337 			}
338 			return m_fallbacks.back.%1$s(args);
339 		}
340 		return m_default.%1$s(args);
341 	}.format(__traits(identifier, func));
342 }
343 
344 private enum PackagesPath = "packages";