From c29727d5241be0fee2265fbbe65d1b275b01dc1c Mon Sep 17 00:00:00 2001 From: Daniel Barkalow Date: Mon, 10 Sep 2007 23:03:11 -0400 Subject: [PATCH] Add fetch methods to transport library. Signed-off-by: Daniel Barkalow Signed-off-by: Junio C Hamano --- transport.c | 284 +++++++++++++++++++++++++++++++++++++++++++++++++++- transport.h | 21 ++++ 2 files changed, 302 insertions(+), 3 deletions(-) diff --git a/transport.c b/transport.c index edbdc3c60..1e7374977 100644 --- a/transport.c +++ b/transport.c @@ -1,6 +1,39 @@ #include "cache.h" #include "transport.h" #include "run-command.h" +#include "http.h" +#include "pkt-line.h" +#include "fetch-pack.h" +#include "walker.h" + +/* Generic functions for using commit walkers */ + +static int fetch_objs_via_walker(const struct transport *transport, + int nr_objs, char **objs) +{ + char *dest = xstrdup(transport->url); + struct walker *walker = transport->data; + + walker->get_all = 1; + walker->get_tree = 1; + walker->get_history = 1; + walker->get_verbosely = transport->verbose; + walker->get_recover = 0; + + if (walker_fetch(walker, nr_objs, objs, NULL, dest)) + die("Fetch failed."); + + free(dest); + return 0; +} + +static int disconnect_walker(struct transport *transport) +{ + struct walker *walker = transport->data; + if (walker) + walker_free(walker); + return 0; +} static const struct transport_ops rsync_transport; @@ -36,9 +69,119 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons return !!err; } +#ifndef NO_CURL +static int missing__target(int code, int result) +{ + return /* file:// URL -- do we ever use one??? */ + (result == CURLE_FILE_COULDNT_READ_FILE) || + /* http:// and https:// URL */ + (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) || + /* ftp:// URL */ + (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE) + ; +} + +#define missing_target(a) missing__target((a)->http_code, (a)->curl_result) + +static struct ref *get_refs_via_curl(const struct transport *transport) +{ + struct buffer buffer; + char *data, *start, *mid; + char *ref_name; + char *refs_url; + int i = 0; + + struct active_request_slot *slot; + struct slot_results results; + + struct ref *refs = NULL; + struct ref *ref = NULL; + struct ref *last_ref = NULL; + + data = xmalloc(4096); + buffer.size = 4096; + buffer.posn = 0; + buffer.buffer = data; + + refs_url = xmalloc(strlen(transport->url) + 11); + sprintf(refs_url, "%s/info/refs", transport->url); + + http_init(); + + slot = get_active_slot(); + slot->results = &results; + curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); + curl_easy_setopt(slot->curl, CURLOPT_URL, refs_url); + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL); + if (start_active_slot(slot)) { + run_active_slot(slot); + if (results.curl_result != CURLE_OK) { + if (missing_target(&results)) { + free(buffer.buffer); + return NULL; + } else { + free(buffer.buffer); + error("%s", curl_errorstr); + return NULL; + } + } + } else { + free(buffer.buffer); + error("Unable to start request"); + return NULL; + } + + http_cleanup(); + + data = buffer.buffer; + start = NULL; + mid = data; + while (i < buffer.posn) { + if (!start) + start = &data[i]; + if (data[i] == '\t') + mid = &data[i]; + if (data[i] == '\n') { + data[i] = 0; + ref_name = mid + 1; + ref = xmalloc(sizeof(struct ref) + + strlen(ref_name) + 1); + memset(ref, 0, sizeof(struct ref)); + strcpy(ref->name, ref_name); + get_sha1_hex(start, ref->old_sha1); + if (!refs) + refs = ref; + if (last_ref) + last_ref->next = ref; + last_ref = ref; + start = NULL; + } + i++; + } + + free(buffer.buffer); + + return refs; +} + +#else + +static struct ref *get_refs_via_curl(const struct transport *transport) +{ + die("Cannot fetch from '%s' without curl ...", transport->url); + return NULL; +} + +#endif + static const struct transport_ops curl_transport = { /* set_option */ NULL, - /* push */ curl_transport_push + /* get_refs_list */ get_refs_via_curl, + /* fetch_refs */ NULL, + /* fetch_objs */ fetch_objs_via_walker, + /* push */ curl_transport_push, + /* disconnect */ disconnect_walker }; static const struct transport_ops bundle_transport = { @@ -46,7 +189,13 @@ static const struct transport_ops bundle_transport = { struct git_transport_data { unsigned thin : 1; + unsigned keep : 1; + + int unpacklimit; + int depth; + + const char *uploadpack; const char *receivepack; }; @@ -54,16 +203,86 @@ static int set_git_option(struct transport *connection, const char *name, const char *value) { struct git_transport_data *data = connection->data; - if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) { + if (!strcmp(name, TRANS_OPT_UPLOADPACK)) { + data->uploadpack = value; + return 0; + } else if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) { data->receivepack = value; return 0; } else if (!strcmp(name, TRANS_OPT_THIN)) { data->thin = !!value; return 0; + } else if (!strcmp(name, TRANS_OPT_KEEP)) { + data->keep = !!value; + return 0; + } else if (!strcmp(name, TRANS_OPT_UNPACKLIMIT)) { + data->unpacklimit = atoi(value); + return 0; + } else if (!strcmp(name, TRANS_OPT_DEPTH)) { + if (!value) + data->depth = 0; + else + data->depth = atoi(value); + return 0; } return 1; } +static struct ref *get_refs_via_connect(const struct transport *transport) +{ + struct git_transport_data *data = transport->data; + struct ref *refs; + int fd[2]; + pid_t pid; + char *dest = xstrdup(transport->url); + + pid = git_connect(fd, dest, data->uploadpack, 0); + + if (pid < 0) + die("Failed to connect to \"%s\"", transport->url); + + get_remote_heads(fd[0], &refs, 0, NULL, 0); + packet_flush(fd[1]); + + finish_connect(pid); + + free(dest); + + return refs; +} + +static int fetch_refs_via_pack(const struct transport *transport, + int nr_heads, char **heads) +{ + struct git_transport_data *data = transport->data; + struct ref *refs; + char *dest = xstrdup(transport->url); + struct fetch_pack_args args; + + args.uploadpack = data->uploadpack; + args.quiet = 0; + args.keep_pack = data->keep; + args.unpacklimit = data->unpacklimit; + args.use_thin_pack = data->thin; + args.fetch_all = 0; + args.verbose = transport->verbose; + args.depth = data->depth; + args.no_progress = 0; + + setup_fetch_pack(&args); + + refs = fetch_pack(dest, nr_heads, heads); + + // ???? check that refs got everything? + + /* free the memory used for the refs list ... */ + + free_refs(refs); + + free(dest); + return 0; +} + static int git_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { struct git_transport_data *data = transport->data; const char **argv; @@ -110,6 +329,9 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const static const struct transport_ops git_transport = { /* set_option */ set_git_option, + /* get_refs_list */ get_refs_via_connect, + /* fetch_refs */ fetch_refs_via_pack, + /* fetch_objs */ NULL, /* push */ git_transport_push }; @@ -140,7 +362,10 @@ struct transport *transport_get(struct remote *remote, const char *url, !prefixcmp(url, "ftp://")) { ret = xmalloc(sizeof(*ret)); ret->ops = &curl_transport; - ret->data = NULL; + if (fetch) + ret->data = get_http_walker(url); + else + ret->data = NULL; } else if (is_local(url) && is_file(url)) { ret = xmalloc(sizeof(*ret)); ret->data = NULL; @@ -150,14 +375,19 @@ struct transport *transport_get(struct remote *remote, const char *url, ret = xcalloc(1, sizeof(*ret)); ret->data = data; data->thin = 1; + data->uploadpack = "git-upload-pack"; + if (remote && remote->uploadpack) + data->uploadpack = remote->uploadpack; data->receivepack = "git-receive-pack"; if (remote && remote->receivepack) data->receivepack = remote->receivepack; + data->unpacklimit = -1; ret->ops = &git_transport; } if (ret) { ret->remote = remote; ret->url = url; + ret->remote_refs = NULL; ret->fetch = !!fetch; } return ret; @@ -186,6 +416,54 @@ int transport_push(struct transport *transport, return transport->ops->push(transport, refspec_nr, refspec, flags); } +struct ref *transport_get_remote_refs(struct transport *transport) +{ + if (!transport->remote_refs) + transport->remote_refs = + transport->ops->get_refs_list(transport); + return transport->remote_refs; +} + +#define PACK_HEADS_CHUNK_COUNT 256 + +int transport_fetch_refs(struct transport *transport, struct ref *refs) +{ + int i; + int nr_heads = 0; + char **heads = xmalloc(PACK_HEADS_CHUNK_COUNT * sizeof(char *)); + struct ref *rm; + int use_objs = !transport->ops->fetch_refs; + + for (rm = refs; rm; rm = rm->next) { + if (rm->peer_ref && + !hashcmp(rm->peer_ref->old_sha1, rm->old_sha1)) + continue; + if (use_objs) { + heads[nr_heads++] = xstrdup(sha1_to_hex(rm->old_sha1)); + } else { + heads[nr_heads++] = xstrdup(rm->name); + } + if (nr_heads % PACK_HEADS_CHUNK_COUNT == 0) + heads = xrealloc(heads, + (nr_heads + PACK_HEADS_CHUNK_COUNT) * + sizeof(char *)); + } + + if (use_objs) { + if (transport->ops->fetch_objs(transport, nr_heads, heads)) + return -1; + } else { + if (transport->ops->fetch_refs(transport, nr_heads, heads)) + return -1; + } + + /* free the memory used for the heads list ... */ + for (i = 0; i < nr_heads; i++) + free(heads[i]); + free(heads); + return 0; +} + int transport_disconnect(struct transport *transport) { int ret = 0; diff --git a/transport.h b/transport.h index 5c2eb959b..5b0a6b065 100644 --- a/transport.h +++ b/transport.h @@ -29,6 +29,11 @@ struct transport_ops { int (*set_option)(struct transport *connection, const char *name, const char *value); + struct ref *(*get_refs_list)(const struct transport *transport); + int (*fetch_refs)(const struct transport *transport, + int nr_refs, char **refs); + int (*fetch_objs)(const struct transport *transport, + int nr_objs, char **objs); int (*push)(struct transport *connection, int refspec_nr, const char **refspec, int flags); int (*disconnect)(struct transport *connection); @@ -40,12 +45,24 @@ struct transport *transport_get(struct remote *remote, const char *url, /* Transport options which apply to git:// and scp-style URLs */ +/* The program to use on the remote side to send a pack */ +#define TRANS_OPT_UPLOADPACK "uploadpack" + /* The program to use on the remote side to receive a pack */ #define TRANS_OPT_RECEIVEPACK "receivepack" /* Transfer the data as a thin pack if not null */ #define TRANS_OPT_THIN "thin" +/* Keep the pack that was transferred if not null */ +#define TRANS_OPT_KEEP "keep" + +/* Unpack the objects if fewer than this number of objects are fetched */ +#define TRANS_OPT_UNPACKLIMIT "unpacklimit" + +/* Limit the depth of the fetch if not null */ +#define TRANS_OPT_DEPTH "depth" + /** * Returns 0 if the option was used, non-zero otherwise. Prints a * message to stderr if the option is not used. @@ -56,6 +73,10 @@ int transport_set_option(struct transport *transport, const char *name, int transport_push(struct transport *connection, int refspec_nr, const char **refspec, int flags); +struct ref *transport_get_remote_refs(struct transport *transport); + +int transport_fetch_refs(struct transport *transport, struct ref *refs); + int transport_disconnect(struct transport *transport); #endif -- 2.26.2