From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 1 Feb 2010 20:41:31 +0000 (-0800)
Subject: Merge branch 'sp/maint-fast-import-large-blob' into sp/fast-import-large-blob
X-Git-Tag: v1.7.0-rc2~24^2
X-Git-Url: http://git.tremily.us/?a=commitdiff_plain;h=844ad3d9a0418b4a3a6a944a50d65403121a0eb6;p=git.git

Merge branch 'sp/maint-fast-import-large-blob' into sp/fast-import-large-blob

* sp/maint-fast-import-large-blob:
  fast-import: Stream very large blobs directly to pack
  bash: don't offer remote transport helpers as subcommands

Conflicts:
	fast-import.c
---

844ad3d9a0418b4a3a6a944a50d65403121a0eb6
diff --cc fast-import.c
index 901784fe9,2105310c5..ca210822d
--- a/fast-import.c
+++ b/fast-import.c
@@@ -2595,140 -2552,39 +2727,142 @@@ static void parse_progress(void
  	skip_optional_lf();
  }
  
 -static void import_marks(const char *input_file)
 +static char* make_fast_import_path(const char *path)
  {
 -	char line[512];
 -	FILE *f = fopen(input_file, "r");
 -	if (!f)
 -		die_errno("cannot read '%s'", input_file);
 -	while (fgets(line, sizeof(line), f)) {
 -		uintmax_t mark;
 -		char *end;
 -		unsigned char sha1[20];
 -		struct object_entry *e;
 +	struct strbuf abs_path = STRBUF_INIT;
  
 -		end = strchr(line, '\n');
 -		if (line[0] != ':' || !end)
 -			die("corrupt mark line: %s", line);
 -		*end = 0;
 -		mark = strtoumax(line + 1, &end, 10);
 -		if (!mark || end == line + 1
 -			|| *end != ' ' || get_sha1(end + 1, sha1))
 -			die("corrupt mark line: %s", line);
 -		e = find_object(sha1);
 -		if (!e) {
 -			enum object_type type = sha1_object_info(sha1, NULL);
 -			if (type < 0)
 -				die("object not found: %s", sha1_to_hex(sha1));
 -			e = insert_object(sha1);
 -			e->type = type;
 -			e->pack_id = MAX_PACK_ID;
 -			e->offset = 1; /* just not zero! */
 -		}
 -		insert_mark(mark, e);
 +	if (!relative_marks_paths || is_absolute_path(path))
 +		return xstrdup(path);
 +	strbuf_addf(&abs_path, "%s/info/fast-import/%s", get_git_dir(), path);
 +	return strbuf_detach(&abs_path, NULL);
 +}
 +
 +static void option_import_marks(const char *marks, int from_stream)
 +{
 +	if (import_marks_file) {
 +		if (from_stream)
 +			die("Only one import-marks command allowed per stream");
 +
 +		/* read previous mark file */
 +		if(!import_marks_file_from_stream)
 +			read_marks();
  	}
 -	fclose(f);
 +
 +	import_marks_file = make_fast_import_path(marks);
 +	import_marks_file_from_stream = from_stream;
 +}
 +
 +static void option_date_format(const char *fmt)
 +{
 +	if (!strcmp(fmt, "raw"))
 +		whenspec = WHENSPEC_RAW;
 +	else if (!strcmp(fmt, "rfc2822"))
 +		whenspec = WHENSPEC_RFC2822;
 +	else if (!strcmp(fmt, "now"))
 +		whenspec = WHENSPEC_NOW;
 +	else
 +		die("unknown --date-format argument %s", fmt);
 +}
 +
 +static void option_max_pack_size(const char *packsize)
 +{
 +	max_packsize = strtoumax(packsize, NULL, 0) * 1024 * 1024;
 +}
 +
 +static void option_depth(const char *depth)
 +{
 +	max_depth = strtoul(depth, NULL, 0);
 +	if (max_depth > MAX_DEPTH)
 +		die("--depth cannot exceed %u", MAX_DEPTH);
 +}
 +
 +static void option_active_branches(const char *branches)
 +{
 +	max_active_branches = strtoul(branches, NULL, 0);
 +}
 +
 +static void option_export_marks(const char *marks)
 +{
 +	export_marks_file = make_fast_import_path(marks);
 +}
 +
 +static void option_export_pack_edges(const char *edges)
 +{
 +	if (pack_edges)
 +		fclose(pack_edges);
 +	pack_edges = fopen(edges, "a");
 +	if (!pack_edges)
 +		die_errno("Cannot open '%s'", edges);
 +}
 +
 +static int parse_one_option(const char *option)
 +{
 +	if (!prefixcmp(option, "max-pack-size=")) {
 +		option_max_pack_size(option + 14);
++	} else if (!prefixcmp(option, "big-file-threshold=")) {
++		big_file_threshold = strtoumax(option + 19, NULL, 0) * 1024 * 1024;
 +	} else if (!prefixcmp(option, "depth=")) {
 +		option_depth(option + 6);
 +	} else if (!prefixcmp(option, "active-branches=")) {
 +		option_active_branches(option + 16);
 +	} else if (!prefixcmp(option, "export-pack-edges=")) {
 +		option_export_pack_edges(option + 18);
 +	} else if (!prefixcmp(option, "quiet")) {
 +		show_stats = 0;
 +	} else if (!prefixcmp(option, "stats")) {
 +		show_stats = 1;
 +	} else {
 +		return 0;
 +	}
 +
 +	return 1;
 +}
 +
 +static int parse_one_feature(const char *feature, int from_stream)
 +{
 +	if (!prefixcmp(feature, "date-format=")) {
 +		option_date_format(feature + 12);
 +	} else if (!prefixcmp(feature, "import-marks=")) {
 +		option_import_marks(feature + 13, from_stream);
 +	} else if (!prefixcmp(feature, "export-marks=")) {
 +		option_export_marks(feature + 13);
 +	} else if (!prefixcmp(feature, "relative-marks")) {
 +		relative_marks_paths = 1;
 +	} else if (!prefixcmp(feature, "no-relative-marks")) {
 +		relative_marks_paths = 0;
 +	} else if (!prefixcmp(feature, "force")) {
 +		force_update = 1;
 +	} else {
 +		return 0;
 +	}
 +
 +	return 1;
 +}
 +
 +static void parse_feature(void)
 +{
 +	char *feature = command_buf.buf + 8;
 +
 +	if (seen_data_command)
 +		die("Got feature command '%s' after data command", feature);
 +
 +	if (parse_one_feature(feature, 1))
 +		return;
 +
 +	die("This version of fast-import does not support feature %s.", feature);
 +}
 +
 +static void parse_option(void)
 +{
 +	char *option = command_buf.buf + 11;
 +
 +	if (seen_data_command)
 +		die("Got option command '%s' after data command", option);
 +
 +	if (parse_one_option(option))
 +		return;
 +
 +	die("This version of fast-import does not support option: %s", option);
  }
  
  static int git_pack_config(const char *k, const char *v, void *cb)
@@@ -2753,37 -2613,11 +2891,37 @@@
  }
  
  static const char fast_import_usage[] =
- "git fast-import [--date-format=f] [--max-pack-size=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]";
+ "git fast-import [--date-format=f] [--max-pack-size=n] [--big-file-threshold=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]";
  
 +static void parse_argv(void)
 +{
 +	unsigned int i;
 +
 +	for (i = 1; i < global_argc; i++) {
 +		const char *a = global_argv[i];
 +
 +		if (*a != '-' || !strcmp(a, "--"))
 +			break;
 +
 +		if (parse_one_option(a + 2))
 +			continue;
 +
 +		if (parse_one_feature(a + 2, 0))
 +			continue;
 +
 +		die("unknown option %s", a);
 +	}
 +	if (i != global_argc)
 +		usage(fast_import_usage);
 +
 +	seen_data_command = 1;
 +	if (import_marks_file)
 +		read_marks();
 +}
 +
  int main(int argc, const char **argv)
  {
 -	unsigned int i, show_stats = 1;
 +	unsigned int i;
  
  	git_extract_argv0_path(argv[0]);
  
diff --cc t/t9300-fast-import.sh
index 60d6f5d1b,513db86ad..131f03298
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@@ -1292,248 -1251,53 +1292,294 @@@ test_expect_success 
  
  echo "$note3_data" >expect
  test_expect_success \
 -	'Q: verify note for third commit' \
 -	'git cat-file blob refs/notes/foobar:$commit3 >actual && test_cmp expect actual'
 +	'Q: verify first note for third commit' \
 +	'git cat-file blob refs/notes/foobar~2:$commit3 >actual && test_cmp expect actual'
 +
 +cat >expect <<EOF
 +parent `git rev-parse --verify refs/notes/foobar~2`
 +author $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +
 +notes (:10)
 +EOF
 +test_expect_success \
 +	'Q: verify second notes commit' \
 +	'git cat-file commit refs/notes/foobar^ | sed 1d >actual &&
 +	test_cmp expect actual'
 +
 +cat >expect.unsorted <<EOF
 +100644 blob $commit1
 +100644 blob $commit2
 +100644 blob $commit3
 +EOF
 +cat expect.unsorted | sort >expect
 +test_expect_success \
 +	'Q: verify second notes tree' \
 +	'git cat-file -p refs/notes/foobar^^{tree} | sed "s/ [0-9a-f]*	/ /" >actual &&
 +	 test_cmp expect actual'
 +
 +echo "$note1b_data" >expect
 +test_expect_success \
 +	'Q: verify second note for first commit' \
 +	'git cat-file blob refs/notes/foobar^:$commit1 >actual && test_cmp expect actual'
 +
 +echo "$note2_data" >expect
 +test_expect_success \
 +	'Q: verify first note for second commit' \
 +	'git cat-file blob refs/notes/foobar^:$commit2 >actual && test_cmp expect actual'
 +
 +echo "$note3_data" >expect
 +test_expect_success \
 +	'Q: verify first note for third commit' \
 +	'git cat-file blob refs/notes/foobar^:$commit3 >actual && test_cmp expect actual'
 +
 +cat >expect <<EOF
 +author $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +
 +notes (:11)
 +EOF
 +test_expect_success \
 +	'Q: verify third notes commit' \
 +	'git cat-file commit refs/notes/foobar2 | sed 1d >actual &&
 +	test_cmp expect actual'
 +
 +cat >expect.unsorted <<EOF
 +100644 blob $commit1
 +EOF
 +cat expect.unsorted | sort >expect
 +test_expect_success \
 +	'Q: verify third notes tree' \
 +	'git cat-file -p refs/notes/foobar2^{tree} | sed "s/ [0-9a-f]*	/ /" >actual &&
 +	 test_cmp expect actual'
 +
 +echo "$note1c_data" >expect
 +test_expect_success \
 +	'Q: verify third note for first commit' \
 +	'git cat-file blob refs/notes/foobar2:$commit1 >actual && test_cmp expect actual'
 +
 +cat >expect <<EOF
 +parent `git rev-parse --verify refs/notes/foobar^`
 +author $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +
 +notes (:12)
 +EOF
 +test_expect_success \
 +	'Q: verify fourth notes commit' \
 +	'git cat-file commit refs/notes/foobar | sed 1d >actual &&
 +	test_cmp expect actual'
 +
 +cat >expect.unsorted <<EOF
 +100644 blob $commit2
 +EOF
 +cat expect.unsorted | sort >expect
 +test_expect_success \
 +	'Q: verify fourth notes tree' \
 +	'git cat-file -p refs/notes/foobar^{tree} | sed "s/ [0-9a-f]*	/ /" >actual &&
 +	 test_cmp expect actual'
 +
 +echo "$note2b_data" >expect
 +test_expect_success \
 +	'Q: verify second note for second commit' \
 +	'git cat-file blob refs/notes/foobar:$commit2 >actual && test_cmp expect actual'
 +
 +###
 +### series R (feature and option)
 +###
 +
 +cat >input <<EOF
 +feature no-such-feature-exists
 +EOF
 +
 +test_expect_success 'R: abort on unsupported feature' '
 +	test_must_fail git fast-import <input
 +'
 +
 +cat >input <<EOF
 +feature date-format=now
 +EOF
 +
 +test_expect_success 'R: supported feature is accepted' '
 +	git fast-import <input
 +'
 +
 +cat >input << EOF
 +blob
 +data 3
 +hi
 +feature date-format=now
 +EOF
 +
 +test_expect_success 'R: abort on receiving feature after data command' '
 +	test_must_fail git fast-import <input
 +'
 +
 +cat >input << EOF
 +feature import-marks=git.marks
 +feature import-marks=git2.marks
 +EOF
 +
 +test_expect_success 'R: only one import-marks feature allowed per stream' '
 +	test_must_fail git fast-import <input
 +'
 +
 +cat >input << EOF
 +feature export-marks=git.marks
 +blob
 +mark :1
 +data 3
 +hi
 +
 +EOF
 +
 +test_expect_success \
 +    'R: export-marks feature results in a marks file being created' \
 +    'cat input | git fast-import &&
 +    grep :1 git.marks'
 +
 +test_expect_success \
 +    'R: export-marks options can be overriden by commandline options' \
 +    'cat input | git fast-import --export-marks=other.marks &&
 +    grep :1 other.marks'
 +
 +cat >input << EOF
 +feature import-marks=marks.out
 +feature export-marks=marks.new
 +EOF
 +
 +test_expect_success \
 +    'R: import to output marks works without any content' \
 +    'cat input | git fast-import &&
 +    test_cmp marks.out marks.new'
 +
 +cat >input <<EOF
 +feature import-marks=nonexistant.marks
 +feature export-marks=marks.new
 +EOF
 +
 +test_expect_success \
 +    'R: import marks prefers commandline marks file over the stream' \
 +    'cat input | git fast-import --import-marks=marks.out &&
 +    test_cmp marks.out marks.new'
 +
 +
 +cat >input <<EOF
 +feature import-marks=nonexistant.marks
 +feature export-marks=combined.marks
 +EOF
 +
 +test_expect_success 'R: multiple --import-marks= should be honoured' '
 +    head -n2 marks.out > one.marks &&
 +    tail -n +3 marks.out > two.marks &&
 +    git fast-import --import-marks=one.marks --import-marks=two.marks <input &&
 +    test_cmp marks.out combined.marks
 +'
 +
 +cat >input <<EOF
 +feature relative-marks
 +feature import-marks=relative.in
 +feature export-marks=relative.out
 +EOF
 +
 +test_expect_success 'R: feature relative-marks should be honoured' '
 +    mkdir -p .git/info/fast-import/ &&
 +    cp marks.new .git/info/fast-import/relative.in &&
 +    git fast-import <input &&
 +    test_cmp marks.new .git/info/fast-import/relative.out
 +'
 +
 +cat >input <<EOF
 +feature relative-marks
 +feature import-marks=relative.in
 +feature no-relative-marks
 +feature export-marks=non-relative.out
 +EOF
 +
 +test_expect_success 'R: feature no-relative-marks should be honoured' '
 +    git fast-import <input &&
 +    test_cmp marks.new non-relative.out
 +'
 +
 +cat >input << EOF
 +option git quiet
 +blob
 +data 3
 +hi
 +
 +EOF
 +
 +touch empty
 +
 +test_expect_success 'R: quiet option results in no stats being output' '
 +    cat input | git fast-import 2> output &&
 +    test_cmp empty output
 +'
 +
 +cat >input <<EOF
 +option git non-existing-option
 +EOF
 +
 +test_expect_success 'R: die on unknown option' '
 +    test_must_fail git fast-import <input
 +'
 +
 +test_expect_success 'R: unknown commandline options are rejected' '\
 +    test_must_fail git fast-import --non-existing-option < /dev/null
 +'
 +
 +cat >input <<EOF
 +option non-existing-vcs non-existing-option
 +EOF
 +
 +test_expect_success 'R: ignore non-git options' '
 +    git fast-import <input
 +'
  
+ ##
+ ## R: very large blobs
+ ##
+ blobsize=$((2*1024*1024 + 53))
+ test-genrandom bar $blobsize >expect
+ cat >input <<INPUT_END
+ commit refs/heads/big-file
+ committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ R - big file
+ COMMIT
+ 
+ M 644 inline big1
+ data $blobsize
+ INPUT_END
+ cat expect >>input
+ cat >>input <<INPUT_END
+ M 644 inline big2
+ data $blobsize
+ INPUT_END
+ cat expect >>input
+ echo >>input
+ 
+ test_expect_success \
+ 	'R: blob bigger than threshold' \
+ 	'test_create_repo R &&
+ 	 git --git-dir=R/.git fast-import --big-file-threshold=1 <input'
+ test_expect_success \
+ 	'R: verify created pack' \
+ 	': >verify &&
+ 	 for p in R/.git/objects/pack/*.pack;
+ 	 do
+ 	   git verify-pack -v $p >>verify || exit;
+ 	 done'
+ test_expect_success \
+ 	'R: verify written objects' \
+ 	'git --git-dir=R/.git cat-file blob big-file:big1 >actual &&
+ 	 test_cmp expect actual &&
+ 	 a=$(git --git-dir=R/.git rev-parse big-file:big1) &&
+ 	 b=$(git --git-dir=R/.git rev-parse big-file:big2) &&
+ 	 test $a = $b'
+ test_expect_success \
+ 	'R: blob appears only once' \
+ 	'n=$(grep $a verify | wc -l) &&
+ 	 test 1 = $n'
+ 
  test_done