[PATCH] Parallelize the pull algorithm

This processes objects in two simultaneous passes. Each object will
first be given to prefetch(), as soon as it is possible to tell that
it will be needed, and then will be given to fetch(), when it is the
next object that needs to be parsed. Unless an implementation does
something with prefetch(), this should have no effect.

Signed-off-by: Daniel Barkalow <barkalow@iabervon.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/http-pull.c b/http-pull.c
index bd6195537ff38375aae3b16391b33c9e4fa23368..e70ab39c03dc82e7f90afa43f8271ca4e3ae8206 100644 (file)
--- a/http-pull.c
+++ b/http-pull.c
@@ -67,6 +67,10 @@ static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
        return size;
 }
 
+void prefetch(unsigned char *sha1)
+{
+}
+
 static int got_indices = 0;
 
 static struct packed_git *packs = NULL;

diff --git a/local-pull.c b/local-pull.c
index 29ae1663958317b612ff5f26707c16612e8b91bc..b715203f6fb482cce8b00cc99c72384e42528b36 100644 (file)
--- a/local-pull.c
+++ b/local-pull.c
@@ -11,6 +11,10 @@ static int use_filecopy = 1;
 
 static char *path; /* "Remote" git repository */
 
+void prefetch(unsigned char *sha1)
+{
+}
+
 int fetch(unsigned char *sha1)
 {
        static int object_name_start = -1;

diff --git a/pull.c b/pull.c
index ed3078e3b27c62c07558fd94f339801cbd685593..950d9f47bd03516ec9e25bfc6742f8582a4197ab 100644 (file)
--- a/pull.c
+++ b/pull.c
@@ -17,11 +17,8 @@ int get_all = 0;
 int get_verbosely = 0;
 static unsigned char current_commit_sha1[20];
 
-static const char commitS[] = "commit";
-static const char treeS[] = "tree";
-static const char blobS[] = "blob";
-
-void pull_say(const char *fmt, const char *hex) {
+void pull_say(const char *fmt, const char *hex) 
+{
        if (get_verbosely)
                fprintf(stderr, fmt, hex);
 }
@@ -48,93 +45,118 @@ static int make_sure_we_have_it(const char *what, unsigned char *sha1)
        return status;
 }
 
-static int process_unknown(unsigned char *sha1);
+static int process(unsigned char *sha1, const char *type);
 
-static int process_tree(unsigned char *sha1)
+static int process_tree(struct tree *tree)
 {
-       struct tree *tree = lookup_tree(sha1);
        struct tree_entry_list *entries;
 
        if (parse_tree(tree))
                return -1;
 
        for (entries = tree->entries; entries; entries = entries->next) {
-               const char *what = entries->directory ? treeS : blobS;
-               if (make_sure_we_have_it(what, entries->item.tree->object.sha1))
+               if (process(entries->item.any->sha1,
+                           entries->directory ? tree_type : blob_type))
                        return -1;
-               if (entries->directory) {
-                       if (process_tree(entries->item.tree->object.sha1))
-                               return -1;
-               }
        }
        return 0;
 }
 
-static int process_commit(unsigned char *sha1)
+static int process_commit(struct commit *commit)
 {
-       struct commit *obj = lookup_commit(sha1);
-
-       if (make_sure_we_have_it(commitS, sha1))
+       if (parse_commit(commit))
                return -1;
 
-       if (parse_commit(obj))
-               return -1;
+       memcpy(current_commit_sha1, commit->object.sha1, 20);
 
        if (get_tree) {
-               if (make_sure_we_have_it(treeS, obj->tree->object.sha1))
-                       return -1;
-               if (process_tree(obj->tree->object.sha1))
+               if (process(commit->tree->object.sha1, tree_type))
                        return -1;
                if (!get_all)
                        get_tree = 0;
        }
        if (get_history) {
-               struct commit_list *parents = obj->parents;
+               struct commit_list *parents = commit->parents;
                for (; parents; parents = parents->next) {
                        if (has_sha1_file(parents->item->object.sha1))
                                continue;
-                       if (make_sure_we_have_it(NULL,
-                                                parents->item->object.sha1)) {
-                               /* The server might not have it, and
-                                * we don't mind. 
-                                */
-                               continue;
-                       }
-                       if (process_commit(parents->item->object.sha1))
+                       if (process(parents->item->object.sha1,
+                                   commit_type))
                                return -1;
-                       memcpy(current_commit_sha1, sha1, 20);
                }
        }
        return 0;
 }
 
-static int process_tag(unsigned char *sha1)
+static int process_tag(struct tag *tag)
 {
-       struct tag *obj = lookup_tag(sha1);
-
-       if (parse_tag(obj))
+       if (parse_tag(tag))
                return -1;
-       return process_unknown(obj->tagged->sha1);
+       return process(tag->tagged->sha1, NULL);
 }
 
-static int process_unknown(unsigned char *sha1)
+static struct object_list *process_queue = NULL;
+static struct object_list **process_queue_end = &process_queue;
+
+static int process(unsigned char *sha1, const char *type)
 {
        struct object *obj;
-       if (make_sure_we_have_it("object", sha1))
-               return -1;
-       obj = parse_object(sha1);
-       if (!obj)
-               return error("Unable to parse object %s", sha1_to_hex(sha1));
-       if (obj->type == commit_type)
-               return process_commit(sha1);
-       if (obj->type == tree_type)
-               return process_tree(sha1);
-       if (obj->type == blob_type)
+       if (has_sha1_file(sha1))
                return 0;
-       if (obj->type == tag_type)
-               return process_tag(sha1);
-       return error("Unable to determine requirement of type %s for %s",
-                    obj->type, sha1_to_hex(sha1));
+       obj = lookup_object_type(sha1, type);
+       if (object_list_contains(process_queue, obj))
+               return 0;
+       object_list_insert(obj, process_queue_end);
+       process_queue_end = &(*process_queue_end)->next;
+
+       //fprintf(stderr, "prefetch %s\n", sha1_to_hex(sha1));
+       prefetch(sha1);
+               
+       return 0;
+}
+
+static int loop(void)
+{
+       while (process_queue) {
+               struct object *obj = process_queue->item;
+               /*
+               fprintf(stderr, "%d objects to pull\n", 
+                       object_list_length(process_queue));
+               */
+               process_queue = process_queue->next;
+               if (!process_queue)
+                       process_queue_end = &process_queue;
+
+               //fprintf(stderr, "fetch %s\n", sha1_to_hex(obj->sha1));
+               
+               if (make_sure_we_have_it(obj->type ?: "object", 
+                                        obj->sha1))
+                       return -1;
+               if (!obj->type)
+                       parse_object(obj->sha1);
+               if (obj->type == commit_type) {
+                       if (process_commit((struct commit *)obj))
+                               return -1;
+                       continue;
+               }
+               if (obj->type == tree_type) {
+                       if (process_tree((struct tree *)obj))
+                               return -1;
+                       continue;
+               }
+               if (obj->type == blob_type) {
+                       continue;
+               }
+               if (obj->type == tag_type) {
+                       if (process_tag((struct tag *)obj))
+                               return -1;
+                       continue;
+               }
+               return error("Unable to determine requirements "
+                            "of type %s for %s",
+                            obj->type, sha1_to_hex(obj->sha1));
+       }
+       return 0;
 }
 
 static int interpret_target(char *target, unsigned char *sha1)
@@ -164,7 +186,9 @@ int pull(char *target)
        if (interpret_target(target, sha1))
                return error("Could not interpret %s as something to pull",
                             target);
-       if (process_unknown(sha1))
+       if (process(sha1, NULL))
+               return -1;
+       if (loop())
                return -1;
        
        if (write_ref) {

diff --git a/pull.h b/pull.h
index e173ae3337c4465da87d849f4e5c9da203fdf01d..e7710e89e710979283f82fe3e67ca1296e70b556 100644 (file)
--- a/pull.h
+++ b/pull.h
@@ -8,6 +8,13 @@
  */
 extern int fetch(unsigned char *sha1);
 
+/*
+ * Fetch the specified object and store it locally; fetch() will be
+ * called later to determine success. To be provided by the particular
+ * implementation.
+ */
+extern void prefetch(unsigned char *sha1);
+
 /*
  * Fetch ref (relative to $GIT_DIR/refs) from the remote, and store
  * the 20-byte SHA1 in sha1.  Return 0 on success, -1 on failure.  To

diff --git a/ssh-pull.c b/ssh-pull.c
index 26356dd7d84ea1bc9f7320b18562ed4117d4fac0..4cf9b6a1677542c74c99e88281417e3cc0d52bb1 100644 (file)
--- a/ssh-pull.c
+++ b/ssh-pull.c
@@ -10,6 +10,10 @@ static int fd_out;
 static unsigned char remote_version = 0;
 static unsigned char local_version = 1;
 
+void prefetch(unsigned char *sha1)
+{
+}
+
 int fetch(unsigned char *sha1)
 {
        int ret;