remote-helpers: Fetch more than one ref in a batch

Some network protocols (e.g. native git://) are able to fetch more
than one ref at a time and reduce the overall transfer cost by
combining the requests into a single exchange.  Instead of feeding
each fetch request one at a time to the helper, feed all of them
at once so the helper can decide whether or not it should batch them.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
CC: Daniel Barkalow <barkalow@iabervon.org>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-remote-helpers.txt b/Documentation/git-remote-helpers.txt
index 173ee232f2cd24231ab95e3337da555be27a4746..e44d821c7b92bbb0d030060ae3ecdba45a830be9 100644 (file)
--- a/Documentation/git-remote-helpers.txt
+++ b/Documentation/git-remote-helpers.txt
@@ -36,10 +36,16 @@ Commands are given by the caller on the helper's standard input, one per line.
        complete list, outputs a blank line.
 
 'fetch' <sha1> <name>::
-       Fetches the given object, writing the necessary objects to the
-       database. Outputs a blank line when the fetch is
-       complete. Only objects which were reported in the ref list
-       with a sha1 may be fetched this way.
+       Fetches the given object, writing the necessary objects
+       to the database.  Fetch commands are sent in a batch, one
+       per line, and the batch is terminated with a blank line.
+       Outputs a single blank line when all fetch commands in the
+       same batch are complete. Only objects which were reported
+       in the ref list with a sha1 may be fetched this way.
++
+Optionally may output a 'lock <file>' line indicating a file under
+GIT_DIR/objects/pack which is keeping a pack until refs can be
+suitably updated.
 +
 Supported if the helper has the "fetch" capability.
 

diff --git a/remote-curl.c b/remote-curl.c
index 478f3ea7d3849d3dccc870ca4e9a6f779e8504c7..22cd5c5fd2a64b0ed4f6a74b38598bbcd2072efa 100644 (file)
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -87,6 +87,81 @@ static struct ref *get_refs(void)
        return refs;
 }
 
+static int fetch_dumb(int nr_heads, struct ref **to_fetch)
+{
+       char **targets = xmalloc(nr_heads * sizeof(char*));
+       int ret, i;
+
+       for (i = 0; i < nr_heads; i++)
+               targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
+
+       init_walker();
+       walker->get_all = 1;
+       walker->get_tree = 1;
+       walker->get_history = 1;
+       walker->get_verbosely = 0;
+       walker->get_recover = 0;
+       ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
+
+       for (i = 0; i < nr_heads; i++)
+               free(targets[i]);
+       free(targets);
+
+       return ret ? error("Fetch failed.") : 0;
+}
+
+static void parse_fetch(struct strbuf *buf)
+{
+       struct ref **to_fetch = NULL;
+       struct ref *list_head = NULL;
+       struct ref **list = &list_head;
+       int alloc_heads = 0, nr_heads = 0;
+
+       do {
+               if (!prefixcmp(buf->buf, "fetch ")) {
+                       char *p = buf->buf + strlen("fetch ");
+                       char *name;
+                       struct ref *ref;
+                       unsigned char old_sha1[20];
+
+                       if (strlen(p) < 40 || get_sha1_hex(p, old_sha1))
+                               die("protocol error: expected sha/ref, got %s'", p);
+                       if (p[40] == ' ')
+                               name = p + 41;
+                       else if (!p[40])
+                               name = "";
+                       else
+                               die("protocol error: expected sha/ref, got %s'", p);
+
+                       ref = alloc_ref(name);
+                       hashcpy(ref->old_sha1, old_sha1);
+
+                       *list = ref;
+                       list = &ref->next;
+
+                       ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
+                       to_fetch[nr_heads++] = ref;
+               }
+               else
+                       die("http transport does not support %s", buf->buf);
+
+               strbuf_reset(buf);
+               if (strbuf_getline(buf, stdin, '\n') == EOF)
+                       return;
+               if (!*buf->buf)
+                       break;
+       } while (1);
+
+       if (fetch_dumb(nr_heads, to_fetch))
+               exit(128); /* error already reported */
+       free_refs(list_head);
+       free(to_fetch);
+
+       printf("\n");
+       fflush(stdout);
+       strbuf_reset(buf);
+}
+
 int main(int argc, const char **argv)
 {
        struct strbuf buf = STRBUF_INIT;
@@ -110,17 +185,8 @@ int main(int argc, const char **argv)
                if (strbuf_getline(&buf, stdin, '\n') == EOF)
                        break;
                if (!prefixcmp(buf.buf, "fetch ")) {
-                       char *obj = buf.buf + strlen("fetch ");
-                       init_walker();
-                       walker->get_all = 1;
-                       walker->get_tree = 1;
-                       walker->get_history = 1;
-                       walker->get_verbosely = 0;
-                       walker->get_recover = 0;
-                       if (walker_fetch(walker, 1, &obj, NULL, NULL))
-                               die("Fetch failed.");
-                       printf("\n");
-                       fflush(stdout);
+                       parse_fetch(&buf);
+
                } else if (!strcmp(buf.buf, "list")) {
                        struct ref *refs = get_refs();
                        struct ref *posn;

diff --git a/transport-helper.c b/transport-helper.c
index f57e84c67641b5a03ba8d9ed5182507bb54dcd15..9de34089478665c498e59d15f16cf0856f0cdac6 100644 (file)
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -10,6 +10,7 @@ struct helper_data
 {
        const char *name;
        struct child_process *helper;
+       FILE *out;
        unsigned fetch : 1;
 };
 
@@ -18,7 +19,6 @@ static struct child_process *get_helper(struct transport *transport)
        struct helper_data *data = transport->data;
        struct strbuf buf = STRBUF_INIT;
        struct child_process *helper;
-       FILE *file;
 
        if (data->helper)
                return data->helper;
@@ -39,9 +39,9 @@ static struct child_process *get_helper(struct transport *transport)
 
        write_str_in_full(helper->in, "capabilities\n");
 
-       file = xfdopen(helper->out, "r");
+       data->out = xfdopen(helper->out, "r");
        while (1) {
-               if (strbuf_getline(&buf, file, '\n') == EOF)
+               if (strbuf_getline(&buf, data->out, '\n') == EOF)
                        exit(128); /* child died, message supplied already */
 
                if (!*buf.buf)
@@ -58,6 +58,7 @@ static int disconnect_helper(struct transport *transport)
        if (data->helper) {
                write_str_in_full(data->helper->in, "\n");
                close(data->helper->in);
+               fclose(data->out);
                finish_command(data->helper);
                free((char *)data->helper->argv[0]);
                free(data->helper->argv);
@@ -70,8 +71,7 @@ static int disconnect_helper(struct transport *transport)
 static int fetch_with_fetch(struct transport *transport,
                            int nr_heads, const struct ref **to_fetch)
 {
-       struct child_process *helper = get_helper(transport);
-       FILE *file = xfdopen(helper->out, "r");
+       struct helper_data *data = transport->data;
        int i;
        struct strbuf buf = STRBUF_INIT;
 
@@ -82,12 +82,30 @@ static int fetch_with_fetch(struct transport *transport,
 
                strbuf_addf(&buf, "fetch %s %s\n",
                            sha1_to_hex(posn->old_sha1), posn->name);
-               write_in_full(helper->in, buf.buf, buf.len);
-               strbuf_reset(&buf);
+       }
 
-               if (strbuf_getline(&buf, file, '\n') == EOF)
+       strbuf_addch(&buf, '\n');
+       if (write_in_full(data->helper->in, buf.buf, buf.len) != buf.len)
+               die_errno("cannot send fetch to %s", data->name);
+
+       while (1) {
+               strbuf_reset(&buf);
+               if (strbuf_getline(&buf, data->out, '\n') == EOF)
                        exit(128); /* child died, message supplied already */
+
+               if (!prefixcmp(buf.buf, "lock ")) {
+                       const char *name = buf.buf + 5;
+                       if (transport->pack_lockfile)
+                               warning("%s also locked %s", data->name, name);
+                       else
+                               transport->pack_lockfile = xstrdup(name);
+               }
+               else if (!buf.len)
+                       break;
+               else
+                       warning("%s unexpectedly said: '%s'", data->name, buf.buf);
        }
+       strbuf_release(&buf);
        return 0;
 }
 
@@ -113,21 +131,20 @@ static int fetch(struct transport *transport,
 
 static struct ref *get_refs_list(struct transport *transport, int for_push)
 {
+       struct helper_data *data = transport->data;
        struct child_process *helper;
        struct ref *ret = NULL;
        struct ref **tail = &ret;
        struct ref *posn;
        struct strbuf buf = STRBUF_INIT;
-       FILE *file;
 
        helper = get_helper(transport);
 
        write_str_in_full(helper->in, "list\n");
 
-       file = xfdopen(helper->out, "r");
        while (1) {
                char *eov, *eon;
-               if (strbuf_getline(&buf, file, '\n') == EOF)
+               if (strbuf_getline(&buf, data->out, '\n') == EOF)
                        exit(128); /* child died, message supplied already */
 
                if (!*buf.buf)