pkgng

Check-in [54821fb118]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Quick'n'dirty implementation of pkg+foo:// URL scheme, for foo in (http, https, ftp). Some style improvements Code in libpkg/fetch.c needs refactoring so we can cleanly implement pkg+ssh:// issue #550
Timelines: family | ancestors | descendants | both | trunk | multi-repos | stash
Files: files | file ages | folders
SHA1: 54821fb11894176395c46f9240a9c5f5edcc6d23
User & Date: m.seaman@infracaninophile.co.uk 2013-09-29 08:53:41
Context
2013-09-29
09:03
etags generates a file called TAGS by default. check-in: 5672d3d33b user: m.seaman@infracaninophile.co.uk tags: trunk, multi-repos, stash
08:53
Quick'n'dirty implementation of pkg+foo:// URL scheme, for foo in (http, https, ftp). Some style improvements Code in libpkg/fetch.c needs refactoring so we can cleanly implement pkg+ssh:// issue #550 check-in: 54821fb118 user: m.seaman@infracaninophile.co.uk tags: trunk, multi-repos, stash
2013-09-27
11:55
When iterating through repos to check for database access, igore the ones which aren't enabled. Add boolean flag to pkg_count_repos: if true, only count the active repos. Fixes #598 check-in: ebae45aa87 user: m.seaman@infracaninophile.co.uk tags: trunk, multi-repos, stash
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to libpkg/fetch.c.

182
183
184
185
186
187
188


189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214

215
216
217
218
219
220
221
222
223
224




























225
226
227
228
229
230
231
...
257
258
259
260
261
262
263






264
265
266
267
268
269
270
...
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
			return (EPKG_OK);
		}
	}
	free(line);
	return (EPKG_FATAL);
}



int
pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t)
{
	FILE *remote = NULL;
	struct url *u;
	struct url_stat st;
	off_t done = 0;
	off_t r;

	int64_t max_retry, retry;
	int64_t fetch_timeout;
	int tmout;
	time_t begin_dl;
	time_t now;
	time_t last = 0;
	char buf[10240];
	char *doc = NULL;
	char docpath[MAXPATHLEN];
	int retcode = EPKG_OK;
	char zone[MAXHOSTNAMELEN + 13];
	struct dns_srvinfo *srv_current = NULL;
	struct http_mirror *http_current = NULL;
	off_t sz = 0;
	int kq = -1, flags = 0;
	struct kevent e, ev;
	struct timespec ts;


	if (pkg_config_int64(PKG_CONFIG_FETCH_RETRY, &max_retry) == EPKG_FATAL)
		max_retry = 3;

	if (pkg_config_int64(PKG_CONFIG_FETCH_TIMEOUT, &fetch_timeout) == EPKG_FATAL)
		fetch_timeout = 30;

	fetchTimeout = (int) fetch_timeout;

	retry = max_retry;





























	u = fetchParseURL(url);
	if (t != NULL)
		u->ims_time = *t;

	if (strcmp(u->scheme, "ssh") == 0) {
		if ((retcode = start_ssh(repo, u, &sz)) != EPKG_OK)
................................................................................

	doc = u->doc;
	while (remote == NULL) {
		if (retry == max_retry) {
			if (repo != NULL && repo->mirror_type == SRV &&
			    (strncmp(u->scheme, "http", 4) == 0
			     || strcmp(u->scheme, "ftp") == 0)) {






				snprintf(zone, sizeof(zone),
				    "_%s._tcp.%s", u->scheme, u->host);
				if (repo->srv == NULL)
					repo->srv = dns_getsrvinfo(zone);
				srv_current = repo->srv;
			} else if (repo != NULL && repo->mirror_type == HTTP &&
			           strncmp(u->scheme, "http", 4) == 0) {
................................................................................
		pkg_emit_error("%s: %s", url, fetchLastErrString);
		retcode = EPKG_FATAL;
		goto cleanup;
	}

	cleanup:

	if (strcmp(u->scheme, "ssh") != 0) {
		if (remote != NULL)
			fclose(remote);
	} else {
		EV_SET(&e, fileno(repo->ssh), EVFILT_READ, EV_DELETE, 0, 0, 0);
		kevent(kq, &e, 1, NULL, 0, NULL);
		flags &= ~O_NONBLOCK;
		if (fcntl(fileno(repo->ssh), F_SETFL, flags) == -1)







>
>



|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>
>
>
>
>







 







|







182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
...
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
...
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
			return (EPKG_OK);
		}
	}
	free(line);
	return (EPKG_FATAL);
}

#define URL_SCHEME_PREFIX	"pkg+"

int
pkg_fetch_file_to_fd(struct pkg_repo *repo, const char *url, int dest, time_t *t)
{
	FILE		*remote = NULL;
	struct url	*u = NULL;
	struct url_stat	 st;
	off_t		 done = 0;
	off_t		 r;

	int64_t		 max_retry, retry;
	int64_t		 fetch_timeout;
	int		 tmout;
	time_t		 begin_dl;
	time_t		 now;
	time_t		 last = 0;
	char		 buf[10240];
	char		*doc = NULL;
	char		 docpath[MAXPATHLEN];
	int		 retcode = EPKG_OK;
	char		 zone[MAXHOSTNAMELEN + 13];
	struct dns_srvinfo	*srv_current = NULL;
	struct http_mirror	*http_current = NULL;
	off_t		 sz = 0;
	int		 kq = -1, flags = 0;
	struct kevent	 e, ev;
	struct timespec	 ts;
	bool		 pkg_url_scheme = false;

	if (pkg_config_int64(PKG_CONFIG_FETCH_RETRY, &max_retry) == EPKG_FATAL)
		max_retry = 3;

	if (pkg_config_int64(PKG_CONFIG_FETCH_TIMEOUT, &fetch_timeout) == EPKG_FATAL)
		fetch_timeout = 30;

	fetchTimeout = (int) fetch_timeout;

	retry = max_retry;

	/* A URL of the form http://host.example.com/ where
	 * host.example.com does not resolve as a simple A record is
	 * not valid according to RFC 2616 Section 3.2.2.  Our usage
	 * with SRV records is incorrect.  However it is encoded into
	 * /usr/sbin/pkg in various releases so we can't just drop it.
         *
         * Instead, introduce new pkg+http://, pkg+https://,
	 * pkg+ssh://, pkg+ftp://, pkg+file:// to support the
	 * SRV-style server discovery, and also to allow eg. Firefox
	 * to run pkg-related stuff given a pkg+foo:// URL.
	 *
	 * Warn if using plain http://, https:// etc with SRV
	 */

	if (strncmp(URL_SCHEME_PREFIX, url, strlen(URL_SCHEME_PREFIX)) == 0) {
		if (repo->mirror_type != SRV) {
			pkg_emit_error("packagesite URL error for %s -- "
				       URL_SCHEME_PREFIX
				       ":// implies SRV mirror type", url);

			/* Too early for there to be anything to cleanup */
			return(EPKG_FATAL);
		}

		url += strlen(URL_SCHEME_PREFIX);
		pkg_url_scheme = true;
	}

	u = fetchParseURL(url);
	if (t != NULL)
		u->ims_time = *t;

	if (strcmp(u->scheme, "ssh") == 0) {
		if ((retcode = start_ssh(repo, u, &sz)) != EPKG_OK)
................................................................................

	doc = u->doc;
	while (remote == NULL) {
		if (retry == max_retry) {
			if (repo != NULL && repo->mirror_type == SRV &&
			    (strncmp(u->scheme, "http", 4) == 0
			     || strcmp(u->scheme, "ftp") == 0)) {

				if (!pkg_url_scheme)
					pkg_emit_notice(
     "Warning: use of %s:// URL scheme with SRV records is deprecated: "
     "switch to pkg+%s://", u->scheme, u->scheme);

				snprintf(zone, sizeof(zone),
				    "_%s._tcp.%s", u->scheme, u->host);
				if (repo->srv == NULL)
					repo->srv = dns_getsrvinfo(zone);
				srv_current = repo->srv;
			} else if (repo != NULL && repo->mirror_type == HTTP &&
			           strncmp(u->scheme, "http", 4) == 0) {
................................................................................
		pkg_emit_error("%s: %s", url, fetchLastErrString);
		retcode = EPKG_FATAL;
		goto cleanup;
	}

	cleanup:

	if (u != NULL && strcmp(u->scheme, "ssh") != 0) {
		if (remote != NULL)
			fclose(remote);
	} else {
		EV_SET(&e, fileno(repo->ssh), EVFILT_READ, EV_DELETE, 0, 0, 0);
		kevent(kq, &e, 1, NULL, 0, NULL);
		flags &= ~O_NONBLOCK;
		if (fcntl(fileno(repo->ssh), F_SETFL, flags) == -1)