index-pack: smarter memory usage when resolving deltas

In the same spirit as commit 9892bebafe, let's avoid allocating the full
buffer for the deflated data in get_data_from_pack() in order to inflate
it.  Let's read and inflate the data in chunks instead to reduce memory
usage.

Signed-off-by: Nicolas Pitre <nico@fluxnic.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin-index-pack.c b/builtin-index-pack.c
index b4cf8c53e0ebbee65a0e4bc0ac1afd1173d1b8e8..127e713ddec575ee15e4ff052711ef2121799a64 100644 (file)
--- a/builtin-index-pack.c
+++ b/builtin-index-pack.c
@@ -359,34 +359,38 @@ static void *get_data_from_pack(struct object_entry *obj)
 {
        off_t from = obj[0].idx.offset + obj[0].hdr_size;
        unsigned long len = obj[1].idx.offset - from;
-       unsigned long rdy = 0;
-       unsigned char *src, *data;
+       unsigned char *data, *inbuf;
        z_stream stream;
-       int st;
+       int status;
 
-       src = xmalloc(len);
-       data = src;
-       do {
-               ssize_t n = pread(pack_fd, data + rdy, len - rdy, from + rdy);
-               if (n < 0)
-                       die_errno("cannot pread pack file");
-               if (!n)
-                       die("premature end of pack file, %lu bytes missing",
-                           len - rdy);
-               rdy += n;
-       } while (rdy < len);
        data = xmalloc(obj->size);
+       inbuf = xmalloc((len < 64*1024) ? len : 64*1024);
+
        memset(&stream, 0, sizeof(stream));
+       git_inflate_init(&stream);
        stream.next_out = data;
        stream.avail_out = obj->size;
-       stream.next_in = src;
-       stream.avail_in = len;
-       git_inflate_init(&stream);
-       while ((st = git_inflate(&stream, Z_FINISH)) == Z_OK);
-       git_inflate_end(&stream);
-       if (st != Z_STREAM_END || stream.total_out != obj->size)
+
+       do {
+               ssize_t n = (len < 64*1024) ? len : 64*1024;
+               n = pread(pack_fd, inbuf, n, from);
+               if (n < 0)
+                       die_errno("cannot pread pack file");
+               if (!n)
+                       die("premature end of pack file, %lu bytes missing", len);
+               from += n;
+               len -= n;
+               stream.next_in = inbuf;
+               stream.avail_in = n;
+               status = git_inflate(&stream, 0);
+       } while (len && status == Z_OK && !stream.avail_in);
+
+       /* This has been inflated OK when first encountered, so... */
+       if (status != Z_STREAM_END || stream.total_out != obj->size)
                die("serious inflate inconsistency");
-       free(src);
+
+       git_inflate_end(&stream);
+       free(inbuf);
        return data;
 }