Lazy man's auto-CRLF

It currently does NOT know about file attributes, so it does its
conversion purely based on content. Maybe that is more in the "git
philosophy" anyway, since content is king, but I think we should try to do
the file attributes to turn it off on demand.

Anyway, BY DEFAULT it is off regardless, because it requires a

	[core]
		AutoCRLF = true

in your config file to be enabled. We could make that the default for
Windows, of course, the same way we do some other things (filemode etc).

But you can actually enable it on UNIX, and it will cause:

 - "git update-index" will write blobs without CRLF
 - "git diff" will diff working tree files without CRLF
 - "git checkout" will write files to the working tree _with_ CRLF

and things work fine.

Funnily, it actually shows an odd file in git itself:

	git clone -n git test-crlf
	cd test-crlf
	git config core.autocrlf true
	git checkout
	git diff

shows a diff for "Documentation/docbook-xsl.css". Why? Because we have
actually checked in that file *with* CRLF! So when "core.autocrlf" is
true, we'll always generate a *different* hash for it in the index,
because the index hash will be for the content _without_ CRLF.

Is this complete? I dunno. It seems to work for me. It doesn't use the
filename at all right now, and that's probably a deficiency (we could
certainly make the "is_binary()" heuristics also take standard filename
heuristics into account).

I don't pass in the filename at all for the "index_fd()" case
(git-update-index), so that would need to be passed around, but this
actually works fine.

NOTE NOTE NOTE! The "is_binary()" heuristics are totally made-up by yours
truly. I will not guarantee that they work at all reasonable. Caveat
emptor. But it _is_ simple, and it _is_ safe, since it's all off by
default.

The patch is pretty simple - the biggest part is the new "convert.c" file,
but even that is really just basic stuff that anybody can write in
"Teaching C 101" as a final project for their first class in programming.
Not to say that it's bug-free, of course - but at least we're not talking
about rocket surgery here.

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/Makefile b/Makefile
index 40bdcff696f3270ced943945183306fc9899407d..60496ff957b153320de139abf3c5ef84554fed42 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -262,7 +262,8 @@ LIB_OBJS = \
        revision.o pager.o tree-walk.o xdiff-interface.o \
        write_or_die.o trace.o list-objects.o grep.o \
        alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \
-       color.o wt-status.o archive-zip.o archive-tar.o shallow.o utf8.o
+       color.o wt-status.o archive-zip.o archive-tar.o shallow.o utf8.o \
+       convert.o
 
 BUILTIN_OBJS = \
        builtin-add.o \

diff --git a/cache.h b/cache.h
index c62b0b090d2b3e628764557127f1629d26459fbf..9c019e8bbac86980c2264414d994d5fbbe844e2e 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -201,6 +201,7 @@ extern const char *apply_default_whitespace;
 extern int zlib_compression_level;
 extern size_t packed_git_window_size;
 extern size_t packed_git_limit;
+extern int auto_crlf;
 
 #define GIT_REPO_VERSION 0
 extern int repository_format_version;
@@ -468,4 +469,8 @@ extern int nfvasprintf(char **str, const char *fmt, va_list va);
 extern void trace_printf(const char *format, ...);
 extern void trace_argv_printf(const char **argv, int count, const char *format, ...);
 
+/* convert.c */
+extern int convert_to_git(const char *path, char **bufp, unsigned long *sizep);
+extern int convert_to_working_tree(const char *path, char **bufp, unsigned long *sizep);
+
 #endif /* CACHE_H */

diff --git a/config.c b/config.c
index d82107124a53bca99ee65ea31dceb3871bf6796f..ffe02129a5a11248cf3a3970c2147c8bdf3e3ea6 100644 (file)
--- a/config.c
+++ b/config.c
@@ -324,6 +324,11 @@ int git_default_config(const char *var, const char *value)
                return 0;
        }
 
+       if (!strcmp(var, "core.autocrlf")) {
+               auto_crlf = git_config_bool(var, value);
+               return 0;
+       }
+
        if (!strcmp(var, "user.name")) {
                strlcpy(git_default_name, value, sizeof(git_default_name));
                return 0;

diff --git a/convert.c b/convert.c
new file mode 100644 (file)
index 0000000..13beb70
--- /dev/null
+++ b/convert.c
@@ -0,0 +1,186 @@
+#include "cache.h"
+/*
+ * convert.c - convert a file when checking it out and checking it in.
+ *
+ * This should use the pathname to decide on whether it wants to do some
+ * more interesting conversions (automatic gzip/unzip, general format
+ * conversions etc etc), but by default it just does automatic CRLF<->LF
+ * translation when the "auto_crlf" option is set.
+ */
+
+struct text_stat {
+       /* CR, LF and CRLF counts */
+       unsigned cr, lf, crlf;
+
+       /* These are just approximations! */
+       unsigned printable, nonprintable;
+};
+
+static void gather_stats(const char *buf, unsigned long size, struct text_stat *stats)
+{
+       unsigned long i;
+
+       memset(stats, 0, sizeof(*stats));
+
+       for (i = 0; i < size; i++) {
+               unsigned char c = buf[i];
+               if (c == '\r') {
+                       stats->cr++;
+                       if (i+1 < size && buf[i+1] == '\n')
+                               stats->crlf++;
+                       continue;
+               }
+               if (c == '\n') {
+                       stats->lf++;
+                       continue;
+               }
+               if (c == 127)
+                       /* DEL */
+                       stats->nonprintable++;
+               else if (c < 32) {
+                       switch (c) {
+                               /* BS, HT, ESC and FF */
+                       case '\b': case '\t': case '\033': case '\014':
+                               stats->printable++;
+                               break;
+                       default:
+                               stats->nonprintable++;
+                       }
+               }
+               else
+                       stats->printable++;
+       }
+}
+
+/*
+ * The same heuristics as diff.c::mmfile_is_binary()
+ */
+static int is_binary(unsigned long size, struct text_stat *stats)
+{
+
+       if ((stats->printable >> 7) < stats->nonprintable)
+               return 1;
+       /*
+        * Other heuristics? Average line length might be relevant,
+        * as might LF vs CR vs CRLF counts..
+        *
+        * NOTE! It might be normal to have a low ratio of CRLF to LF
+        * (somebody starts with a LF-only file and edits it with an editor
+        * that adds CRLF only to lines that are added..). But do  we
+        * want to support CR-only? Probably not.
+        */
+       return 0;
+}
+
+int convert_to_git(const char *path, char **bufp, unsigned long *sizep)
+{
+       char *buffer, *nbuf;
+       unsigned long size, nsize;
+       struct text_stat stats;
+
+       /*
+        * FIXME! Other pluggable conversions should go here,
+        * based on filename patterns. Right now we just do the
+        * stupid auto-CRLF one.
+        */
+       if (!auto_crlf)
+               return 0;
+
+       size = *sizep;
+       if (!size)
+               return 0;
+       buffer = *bufp;
+
+       gather_stats(buffer, size, &stats);
+
+       /* No CR? Nothing to convert, regardless. */
+       if (!stats.cr)
+               return 0;
+
+       /*
+        * We're currently not going to even try to convert stuff
+        * that has bare CR characters. Does anybody do that crazy
+        * stuff?
+        */
+       if (stats.cr != stats.crlf)
+               return 0;
+
+       /*
+        * And add some heuristics for binary vs text, of course...
+        */
+       if (is_binary(size, &stats))
+               return 0;
+
+       /*
+        * Ok, allocate a new buffer, fill it in, and return true
+        * to let the caller know that we switched buffers on it.
+        */
+       nsize = size - stats.crlf;
+       nbuf = xmalloc(nsize);
+       *bufp = nbuf;
+       *sizep = nsize;
+       do {
+               unsigned char c = *buffer++;
+               if (c != '\r')
+                       *nbuf++ = c;
+       } while (--size);
+
+       return 1;
+}
+
+int convert_to_working_tree(const char *path, char **bufp, unsigned long *sizep)
+{
+       char *buffer, *nbuf;
+       unsigned long size, nsize;
+       struct text_stat stats;
+       unsigned char last;
+
+       /*
+        * FIXME! Other pluggable conversions should go here,
+        * based on filename patterns. Right now we just do the
+        * stupid auto-CRLF one.
+        */
+       if (!auto_crlf)
+               return 0;
+
+       size = *sizep;
+       if (!size)
+               return 0;
+       buffer = *bufp;
+
+       gather_stats(buffer, size, &stats);
+
+       /* No LF? Nothing to convert, regardless. */
+       if (!stats.lf)
+               return 0;
+
+       /* Was it already in CRLF format? */
+       if (stats.lf == stats.crlf)
+               return 0;
+
+       /* If we have any bare CR characters, we're not going to touch it */
+       if (stats.cr != stats.crlf)
+               return 0;
+
+       if (is_binary(size, &stats))
+               return 0;
+
+       /*
+        * Ok, allocate a new buffer, fill it in, and return true
+        * to let the caller know that we switched buffers on it.
+        */
+       nsize = size + stats.lf - stats.crlf;
+       nbuf = xmalloc(nsize);
+       *bufp = nbuf;
+       *sizep = nsize;
+       last = 0;
+       do {
+               unsigned char c = *buffer++;
+               if (c == '\n' && last != '\r')
+                       *nbuf++ = '\r';
+               *nbuf++ = c;
+               last = c;
+       } while (--size);
+
+       return 1;
+}

diff --git a/diff.c b/diff.c
index 13b9b6c5602cc1aca4a95ed4d292756d3380c543..561587cace6d94ce2d2c4653588bcba237555f37 100644 (file)
--- a/diff.c
+++ b/diff.c
@@ -1332,6 +1332,9 @@ int diff_populate_filespec(struct diff_filespec *s, int size_only)
            reuse_worktree_file(s->path, s->sha1, 0)) {
                struct stat st;
                int fd;
+               char *buf;
+               unsigned long size;
+
                if (lstat(s->path, &st) < 0) {
                        if (errno == ENOENT) {
                        err_empty:
@@ -1364,7 +1367,19 @@ int diff_populate_filespec(struct diff_filespec *s, int size_only)
                s->data = xmmap(NULL, s->size, PROT_READ, MAP_PRIVATE, fd, 0);
                close(fd);
                s->should_munmap = 1;
-               /* FIXME! CRLF -> LF conversion goes here, based on "s->path" */
+
+               /*
+                * Convert from working tree format to canonical git format
+                */
+               buf = s->data;
+               size = s->size;
+               if (convert_to_git(s->path, &buf, &size)) {
+                       munmap(s->data, s->size);
+                       s->should_munmap = 0;
+                       s->data = buf;
+                       s->size = size;
+                       s->should_free = 1;
+               }
        }
        else {
                char type[20];

diff --git a/entry.c b/entry.c
index c2641ddefdfde5ccb08af900ac48378231f969c7..472a9ef32191f211347a98d4ab7d06a06de043ec 100644 (file)
--- a/entry.c
+++ b/entry.c
@@ -78,6 +78,9 @@ static int write_entry(struct cache_entry *ce, char *path, struct checkout *stat
                        path, sha1_to_hex(ce->sha1));
        }
        switch (ntohl(ce->ce_mode) & S_IFMT) {
+               char *buf;
+               unsigned long nsize;
+
        case S_IFREG:
                if (to_tempfile) {
                        strcpy(path, ".merge_file_XXXXXX");
@@ -89,7 +92,18 @@ static int write_entry(struct cache_entry *ce, char *path, struct checkout *stat
                        return error("git-checkout-index: unable to create file %s (%s)",
                                path, strerror(errno));
                }
-               /* FIXME: LF -> CRLF conversion goes here, based on "ce->name" */
+
+               /*
+                * Convert from git internal format to working tree format
+                */
+               buf = new;
+               nsize = size;
+               if (convert_to_working_tree(ce->name, &buf, &nsize)) {
+                       free(new);
+                       new = buf;
+                       size = nsize;
+               }
+
                wrote = write_in_full(fd, new, size);
                close(fd);
                free(new);

diff --git a/environment.c b/environment.c
index 54c22f8248cc04647829c8a7558f2bb24515f30f..2fa0960412755fe6d4ca93348165e6591d1fbf0d 100644 (file)
--- a/environment.c
+++ b/environment.c
@@ -28,6 +28,7 @@ size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
 size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
 int pager_in_use;
 int pager_use_color = 1;
+int auto_crlf = 0;
 
 static const char *git_dir;
 static char *git_object_dir, *git_index_file, *git_refs_dir, *git_graft_file;

diff --git a/sha1_file.c b/sha1_file.c
index 8ad7fad825708b5a56a3bfd4b8b98708de0bb2dc..6ec67b2923aafdb1e2c0da556e584db0468de9ff 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -2082,7 +2082,7 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, con
 {
        unsigned long size = st->st_size;
        void *buf;
-       int ret;
+       int ret, re_allocated = 0;
 
        buf = "";
        if (size)
@@ -2091,11 +2091,30 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, con
 
        if (!type)
                type = blob_type;
-       /* FIXME: CRLF -> LF conversion here for blobs! We'll need the path! */
+
+       /*
+        * Convert blobs to git internal format
+        */
+       if (!strcmp(type, blob_type)) {
+               unsigned long nsize = size;
+               char *nbuf = buf;
+               if (convert_to_git(NULL, &nbuf, &nsize)) {
+                       if (size)
+                               munmap(buf, size);
+                       size = nsize;
+                       buf = nbuf;
+                       re_allocated = 1;
+               }
+       }
+
        if (write_object)
                ret = write_sha1_file(buf, size, type, sha1);
        else
                ret = hash_sha1_file(buf, size, type, sha1);
+       if (re_allocated) {
+               free(buf);
+               return ret;
+       }
        if (size)
                munmap(buf, size);
        return ret;