index-pack: learn about pack index version 2

Like previous patch but for index-pack.

[ There is quite some code duplication between pack-objects and index-pack
  for generating a pack index (and fast-import as well I suppose).  This
  should be reworked into a common function eventually. But not now. ]

Signed-off-by: Nicolas Pitre <nico@cam.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/index-pack.c b/index-pack.c
index d33f723365719bef2370428a691832243aa7cb37..a833f640f6bb4e9eb2b876b65644b410b426dbb0 100644 (file)
--- a/index-pack.c
+++ b/index-pack.c
@@ -686,9 +686,10 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
 {
        struct sha1file *f;
        struct object_entry **sorted_by_sha, **list, **last;
-       unsigned int array[256];
+       uint32_t array[256];
        int i, fd;
        SHA_CTX ctx;
+       uint32_t index_version;
 
        if (nr_objects) {
                sorted_by_sha =
@@ -699,7 +700,6 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
                        sorted_by_sha[i] = &objects[i];
                qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
                      sha1_compare);
-
        }
        else
                sorted_by_sha = list = last = NULL;
@@ -718,6 +718,17 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
                die("unable to create %s: %s", index_name, strerror(errno));
        f = sha1fd(fd, index_name);
 
+       /* if last object's offset is >= 2^31 we should use index V2 */
+       index_version = (objects[nr_objects-1].offset >> 31) ? 2 : 1;
+
+       /* index versions 2 and above need a header */
+       if (index_version >= 2) {
+               struct pack_idx_header hdr;
+               hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
+               hdr.idx_version = htonl(index_version);
+               sha1write(f, &hdr, sizeof(hdr));
+       }
+
        /*
         * Write the first-level table (the list is sorted,
         * but we use a 256-entry lookup to be able to avoid
@@ -734,24 +745,61 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1)
                array[i] = htonl(next - sorted_by_sha);
                list = next;
        }
-       sha1write(f, array, 256 * sizeof(int));
+       sha1write(f, array, 256 * 4);
 
-       /* recompute the SHA1 hash of sorted object names.
-        * currently pack-objects does not do this, but that
-        * can be fixed.
-        */
+       /* compute the SHA1 hash of sorted object names. */
        SHA1_Init(&ctx);
+
        /*
         * Write the actual SHA1 entries..
         */
        list = sorted_by_sha;
        for (i = 0; i < nr_objects; i++) {
                struct object_entry *obj = *list++;
-               unsigned int offset = htonl(obj->offset);
-               sha1write(f, &offset, 4);
+               if (index_version < 2) {
+                       uint32_t offset = htonl(obj->offset);
+                       sha1write(f, &offset, 4);
+               }
                sha1write(f, obj->sha1, 20);
                SHA1_Update(&ctx, obj->sha1, 20);
        }
+
+       if (index_version >= 2) {
+               unsigned int nr_large_offset = 0;
+
+               /* write the crc32 table */
+               list = sorted_by_sha;
+               for (i = 0; i < nr_objects; i++) {
+                       struct object_entry *obj = *list++;
+                       uint32_t crc32_val = htonl(obj->crc32);
+                       sha1write(f, &crc32_val, 4);
+               }
+
+               /* write the 32-bit offset table */
+               list = sorted_by_sha;
+               for (i = 0; i < nr_objects; i++) {
+                       struct object_entry *obj = *list++;
+                       uint32_t offset = (obj->offset <= 0x7fffffff) ?
+                               obj->offset : (0x80000000 | nr_large_offset++);
+                       offset = htonl(offset);
+                       sha1write(f, &offset, 4);
+               }
+
+               /* write the large offset table */
+               list = sorted_by_sha;
+               while (nr_large_offset) {
+                       struct object_entry *obj = *list++;
+                       uint64_t offset = obj->offset;
+                       if (offset > 0x7fffffff) {
+                               uint32_t split[2];
+                               split[0]        = htonl(offset >> 32);
+                               split[1] = htonl(offset & 0xffffffff);
+                               sha1write(f, split, 8);
+                               nr_large_offset--;
+                       }
+               }
+       }
+
        sha1write(f, sha1, 20);
        sha1close(f, NULL, 1);
        free(sorted_by_sha);