Lazily open pack index files on demand

In some repository configurations the user may have many packfiles,
but all of the recent commits/trees/tags/blobs are likely to
be in the most recent packfile (the one with the newest mtime).
It is therefore common to be able to complete an entire operation
by accessing only one packfile, even if there are 25 packfiles
available to the repository.

Rather than opening and mmaping the corresponding .idx file for
every pack found, we now only open and map the .idx when we suspect
there might be an object of interest in there.

Of course we cannot known in advance which packfile contains an
object, so we still need to scan the entire packed_git list to
locate anything.  But odds are users want to access objects in the
most recently created packfiles first, and that may be all they
ever need for the current operation.

Junio observed in b867092f that placing recent packfiles before
older ones can slightly improve access times for recent objects,
without degrading it for historical object access.

This change improves upon Junio's observations by trying even harder
to avoid the .idx files that we won't need.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/builtin-count-objects.c b/builtin-count-objects.c
index ff90ebd465002882781507ecfcfc33cab2f759fc..ac65e03e7f5203def8c412da95f254973119c14d 100644 (file)
--- a/builtin-count-objects.c
+++ b/builtin-count-objects.c
@@ -111,6 +111,8 @@ int cmd_count_objects(int ac, const char **av, const char *prefix)
                for (p = packed_git; p; p = p->next) {
                        if (!p->pack_local)
                                continue;
+                       if (!p->index_data && open_pack_index(p))
+                               continue;
                        packed += p->num_objects;
                        num_pack++;
                }

diff --git a/cache.h b/cache.h
index cd875bc2e95ded83eb8619eec106fff88a9012c4..0f4a05b51ea7aca66e308443eefa1aa11d822a5f 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -485,10 +485,11 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
                                         struct packed_git *packs);
 
 extern void pack_report(void);
+extern int open_pack_index(struct packed_git *);
 extern unsigned char* use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
 extern void unuse_pack(struct pack_window **);
 extern struct packed_git *add_packed_git(const char *, int, int);
-extern const unsigned char *nth_packed_object_sha1(const struct packed_git *, uint32_t);
+extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t);
 extern off_t find_pack_entry_one(const unsigned char *, struct packed_git *);
 extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *);
 extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);

diff --git a/pack-check.c b/pack-check.c
index c168642c0c0d9479b1ccc8564ce6c80c9055ab00..3623c716e3e3b7d4a28800e34afd0e5f0e13a43d 100644 (file)
--- a/pack-check.c
+++ b/pack-check.c
@@ -128,12 +128,17 @@ static void show_pack_info(struct packed_git *p)
 
 int verify_pack(struct packed_git *p, int verbose)
 {
-       off_t index_size = p->index_size;
-       const unsigned char *index_base = p->index_data;
+       off_t index_size;
+       const unsigned char *index_base;
        SHA_CTX ctx;
        unsigned char sha1[20];
        int ret;
 
+       if (open_pack_index(p))
+               return error("packfile %s index not opened", p->pack_name);
+       index_size = p->index_size;
+       index_base = p->index_data;
+
        ret = 0;
        /* Verify SHA1 sum of the index file */
        SHA1_Init(&ctx);

diff --git a/pack-redundant.c b/pack-redundant.c
index 87077e150c1b53a26464089a2cabafafb32e6636..06173206f09a39b2d9d572d7842f66674264964d 100644 (file)
--- a/pack-redundant.c
+++ b/pack-redundant.c
@@ -550,6 +550,9 @@ static struct pack_list * add_pack(struct packed_git *p)
        l.pack = p;
        llist_init(&l.all_objects);
 
+       if (!p->index_data && open_pack_index(p))
+               return NULL;
+
        base = p->index_data;
        base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
        step = (p->index_version < 2) ? 24 : 20;

diff --git a/sha1_file.c b/sha1_file.c
index 12d2ef201197d93885d0adacd43d9621950a8615..6a5ba635003858d70e449122351b502b823de13a 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -530,6 +530,21 @@ static int check_packed_git_idx(const char *path,  struct packed_git *p)
        return 0;
 }
 
+int open_pack_index (struct packed_git *p)
+{
+       char *idx_name;
+       int ret;
+
+       if (p->index_data)
+               return 0;
+
+       idx_name = xstrdup(p->pack_name);
+       strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
+       ret = check_packed_git_idx(idx_name, p);
+       free(idx_name);
+       return ret;
+}
+
 static void scan_windows(struct packed_git *p,
        struct packed_git **lru_p,
        struct pack_window **lru_w,
@@ -605,6 +620,9 @@ static int open_packed_git_1(struct packed_git *p)
        unsigned char *idx_sha1;
        long fd_flag;
 
+       if (!p->index_data && open_pack_index(p))
+               return error("packfile %s index unavailable", p->pack_name);
+
        p->pack_fd = open(p->pack_name, O_RDONLY);
        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
                return -1;
@@ -757,8 +775,7 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
                return NULL;
        memcpy(p->pack_name, path, path_len);
        strcpy(p->pack_name + path_len, ".pack");
-       if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode) ||
-           check_packed_git_idx(path, p)) {
+       if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
                free(p);
                return NULL;
        }
@@ -766,6 +783,10 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
        /* ok, it looks sane as far as we can check without
         * actually mapping the pack file.
         */
+       p->index_version = 0;
+       p->index_data = NULL;
+       p->index_size = 0;
+       p->num_objects = 0;
        p->pack_size = st.st_size;
        p->next = NULL;
        p->windows = NULL;
@@ -1572,10 +1593,15 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
        return data;
 }
 
-const unsigned char *nth_packed_object_sha1(const struct packed_git *p,
+const unsigned char *nth_packed_object_sha1(struct packed_git *p,
                                            uint32_t n)
 {
        const unsigned char *index = p->index_data;
+       if (!index) {
+               if (open_pack_index(p))
+                       return NULL;
+               index = p->index_data;
+       }
        if (n >= p->num_objects)
                return NULL;
        index += 4 * 256;
@@ -1612,6 +1638,12 @@ off_t find_pack_entry_one(const unsigned char *sha1,
        const unsigned char *index = p->index_data;
        unsigned hi, lo;
 
+       if (!index) {
+               if (open_pack_index(p))
+                       return 0;
+               level1_ofs = p->index_data;
+               index = p->index_data;
+       }
        if (p->index_version > 1) {
                level1_ofs += 2;
                index += 8;