git-pack-objects: write the pack files with a SHA1 csum

We want to be able to check their integrity later, and putting the
sha1-sum of the contents at the end is a good thing.  The writing
routines are generic, so we could try to re-use them for the index file,
instead of having the same logic duplicated.

Update unpack-objects to know about the extra 20 bytes at the end
of the index.

diff --git a/Makefile b/Makefile
index e74b90bfeb3652ff2e8224b5771ce8d3df502e8f..e55d352146700eda4f36d30a777f4713eb3423aa 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -45,9 +45,9 @@ install: $(PROG) $(SCRIPTS)
 
 LIB_OBJS=read-cache.o sha1_file.o usage.o object.o commit.o tree.o blob.o \
         tag.o delta.o date.o index.o diff-delta.o patch-delta.o entry.o \
-        epoch.o refs.o
+        epoch.o refs.o csum-file.o
 LIB_FILE=libgit.a
-LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h
+LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h csum-file.h
 
 LIB_H += strbuf.h
 LIB_OBJS += strbuf.o

diff --git a/csum-file.c b/csum-file.c
new file mode 100644 (file)
index 0000000..98cb59d
--- /dev/null
+++ b/csum-file.c
@@ -0,0 +1,120 @@
+/*
+ * csum-file.c
+ *
+ * Copyright (C) 2005 Linus Torvalds
+ *
+ * Simple file write infrastructure for writing SHA1-summed
+ * files. Useful when you write a file that you want to be
+ * able to verify hasn't been messed with afterwards.
+ */
+#include "cache.h"
+#include "csum-file.h"
+
+static int sha1flush(struct sha1file *f, unsigned int count)
+{
+       void *buf = f->buffer;
+
+       for (;;) {
+               int ret = write(f->fd, buf, count);
+               if (ret > 0) {
+                       buf += ret;
+                       count -= ret;
+                       if (count)
+                               continue;
+                       return 0;
+               }
+               if (!ret)
+                       die("sha1 file write error. Out of diskspace");
+               if (errno == EAGAIN || errno == EINTR)
+                       continue;
+               die("sha1 file write error (%s)", strerror(errno));
+       }
+}
+
+int sha1close(struct sha1file *f)
+{
+       unsigned offset = f->offset;
+       if (offset) {
+               SHA1_Update(&f->ctx, f->buffer, offset);
+               sha1flush(f, offset);
+       }
+       SHA1_Final(f->buffer, &f->ctx);
+       sha1flush(f, 20);
+       return 0;
+}
+
+int sha1write(struct sha1file *f, void *buf, unsigned int count)
+{
+       while (count) {
+               unsigned offset = f->offset;
+               unsigned left = sizeof(f->buffer) - offset;
+               unsigned nr = count > left ? left : count;
+
+               memcpy(f->buffer + offset, buf, nr);
+               count -= nr;
+               offset += nr;
+               left -= nr;
+               if (!left) {
+                       SHA1_Update(&f->ctx, f->buffer, offset);
+                       sha1flush(f, offset);
+                       offset = 0;
+               }
+               f->offset = offset;
+       }
+       return 0;
+}
+
+struct sha1file *sha1create(const char *fmt, ...)
+{
+       static char filename[PATH_MAX];
+       struct sha1file *f;
+       unsigned len;
+       va_list arg;
+       int fd;
+
+       va_start(arg, fmt);
+       len = vsnprintf(filename, PATH_MAX, fmt, arg);
+       va_end(arg);
+
+       if (len >= PATH_MAX)
+               die("you wascally wabbit, you");
+       fd = open(filename, O_CREAT | O_EXCL | O_WRONLY, 0644);
+       if (fd < 0)
+               die("unable to open %s (%s)", filename, strerror(errno));
+       f = xmalloc(sizeof(*f));
+       f->fd = fd;
+       f->error = 0;
+       f->offset = 0;
+       SHA1_Init(&f->ctx);
+       return f;
+}
+
+int sha1write_compressed(struct sha1file *f, void *in, unsigned int size)
+{
+       z_stream stream;
+       unsigned long maxsize;
+       void *out;
+
+       memset(&stream, 0, sizeof(stream));
+       deflateInit(&stream, Z_DEFAULT_COMPRESSION);
+       maxsize = deflateBound(&stream, size);
+       out = xmalloc(maxsize);
+
+       /* Compress it */
+       stream.next_in = in;
+       stream.avail_in = size;
+
+       stream.next_out = out;
+       stream.avail_out = maxsize;
+
+       while (deflate(&stream, Z_FINISH) == Z_OK)
+               /* nothing */;
+       deflateEnd(&stream);
+
+       size = stream.total_out;
+       sha1write(f, out, size);
+       free(out);
+       return size;
+}
+
+

diff --git a/csum-file.h b/csum-file.h
new file mode 100644 (file)
index 0000000..1086f04
--- /dev/null
+++ b/csum-file.h
@@ -0,0 +1,17 @@
+#ifndef CSUM_FILE_H
+#define CSUM_FILE_H
+
+/* A SHA1-protected file */
+struct sha1file {
+       int fd, error;
+       unsigned long offset;
+       SHA_CTX ctx;
+       unsigned char buffer[8192];
+};
+
+extern struct sha1file *sha1create(const char *fmt, ...);
+extern int sha1close(struct sha1file *);
+extern int sha1write(struct sha1file *, void *, unsigned int);
+extern int sha1write_compressed(struct sha1file *, void *, unsigned int);
+
+#endif

diff --git a/pack-objects.c b/pack-objects.c
index bc8bb954bfb7d9ba995e0f22f44361dcb24b9162..f0c84c9a746a3dcf1e3427d33310d2cdaf0bf69a 100644 (file)
--- a/pack-objects.c
+++ b/pack-objects.c
@@ -2,6 +2,7 @@
 #include "cache.h"
 #include "object.h"
 #include "delta.h"
+#include "csum-file.h"
 
 static const char pack_usage[] = "git-pack-objects [--window=N] [--depth=N] base-name < object-list";
 
@@ -29,51 +30,6 @@ static struct object_entry *objects = NULL;
 static int nr_objects = 0, nr_alloc = 0;
 static const char *base_name;
 
-struct myfile {
-       int fd;
-       unsigned long chars;
-       unsigned char buffer[8192];
-};
-
-static FILE *create_file(const char *suffix)
-{
-       static char filename[PATH_MAX];
-       unsigned len;
-
-       len = snprintf(filename, PATH_MAX, "%s.%s", base_name, suffix);
-       if (len >= PATH_MAX)
-               die("you wascally wabbit, you");
-       return fopen(filename, "w");
-}
-
-static unsigned long fwrite_compressed(void *in, unsigned long size, FILE *f)
-{
-       z_stream stream;
-       unsigned long maxsize;
-       void *out;
-
-       memset(&stream, 0, sizeof(stream));
-       deflateInit(&stream, Z_DEFAULT_COMPRESSION);
-       maxsize = deflateBound(&stream, size);
-       out = xmalloc(maxsize);
-
-       /* Compress it */
-       stream.next_in = in;
-       stream.avail_in = size;
-
-       stream.next_out = out;
-       stream.avail_out = maxsize;
-
-       while (deflate(&stream, Z_FINISH) == Z_OK)
-               /* nothing */;
-       deflateEnd(&stream);
-
-       size = stream.total_out;
-       fwrite(out, size, 1, f);
-       free(out);
-       return size;
-}
-
 static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
 {
        unsigned long othersize, delta_size;
@@ -92,7 +48,7 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
        return delta_buf;
 }
 
-static unsigned long write_object(FILE *f, struct object_entry *entry)
+static unsigned long write_object(struct sha1file *f, struct object_entry *entry)
 {
        unsigned long size;
        char type[10];
@@ -121,8 +77,8 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
        }
        datalen = htonl(size);
        memcpy(header+1, &datalen, 4);
-       fwrite(header, hdrlen, 1, f);
-       datalen = fwrite_compressed(buf, size, f);
+       sha1write(f, header, hdrlen);
+       datalen = sha1write_compressed(f, buf, size);
        free(buf);
        return hdrlen + datalen;
 }
@@ -130,7 +86,7 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
 static void write_pack_file(void)
 {
        int i;
-       FILE *f = create_file("pack");
+       struct sha1file *f = sha1create("%s.%s", base_name, "pack");
        unsigned long offset = 0;
        unsigned long mb;
 
@@ -139,7 +95,7 @@ static void write_pack_file(void)
                entry->offset = offset;
                offset += write_object(f, entry);
        }
-       fclose(f);
+       sha1close(f);
        mb = offset >> 20;
        offset &= 0xfffff;
 }
@@ -147,7 +103,7 @@ static void write_pack_file(void)
 static void write_index_file(void)
 {
        int i;
-       FILE *f = create_file("idx");
+       struct sha1file *f = sha1create("%s.%s", base_name, "idx");
        struct object_entry **list = sorted_by_sha;
        struct object_entry **last = list + nr_objects;
        unsigned int array[256];
@@ -168,7 +124,7 @@ static void write_index_file(void)
                array[i] = htonl(next - sorted_by_sha);
                list = next;
        }
-       fwrite(array, 256, sizeof(int), f);
+       sha1write(f, array, 256 * sizeof(int));
 
        /*
         * Write the actual SHA1 entries..
@@ -177,10 +133,10 @@ static void write_index_file(void)
        for (i = 0; i < nr_objects; i++) {
                struct object_entry *entry = *list++;
                unsigned int offset = htonl(entry->offset);
-               fwrite(&offset, 4, 1, f);
-               fwrite(entry->sha1, 20, 1, f);
+               sha1write(f, &offset, 4);
+               sha1write(f, entry->sha1, 20);
        }
-       fclose(f);
+       sha1close(f);
 }
 
 static void add_object_entry(unsigned char *sha1, unsigned int hash)

diff --git a/unpack-objects.c b/unpack-objects.c
index 9da3ac89a85abc71cc1b2ec9159acb119ba23009..91a71c55c6bb720c7fbe3d1d2bd393d5d22f764f 100644 (file)
--- a/unpack-objects.c
+++ b/unpack-objects.c
@@ -61,7 +61,7 @@ static int check_index(void)
        unsigned int nr;
        int i;
 
-       if (index_size < 4*256)
+       if (index_size < 4*256 + 20)
                return error("index file too small");
        nr = 0;
        for (i = 0; i < 256; i++) {
@@ -70,11 +70,14 @@ static int check_index(void)
                        return error("non-monotonic index");
                nr = n;
        }
-       if (index_size != 4*256 + nr * 24) {
-               printf("index_size=%lu, expected %u (%u)\n",
-                      index_size, 4*256 + nr * 24, nr);
+       /*
+        * Total size:
+        *  - 256 index entries 4 bytes each
+        *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
+        *  - 20-byte SHA1 file checksum
+        */
+       if (index_size != 4*256 + nr * 24 + 20)
                return error("wrong index file size");
-       }
 
        nr_entries = nr;
        pack_list = xmalloc(nr * sizeof(struct pack_entry *));