SVN import: Use one log call

One "svn log" (or its equivalent) per revision adds delay and server load.
Instead, open two SVN connections -- one for the log, and one for the files.

Positive side effect: Only those log entries which actually contain data
are committed => no more empty commits.

Also, change the "-l" option to set the maximum revision to be pulled,
not the number of revisions.

Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/Documentation/git-svnimport.txt b/Documentation/git-svnimport.txt
index fcc79fa93a18444e5df42bc9e98bb18fe4b55ce5..f8dbee70961f932e1c2be29bd11d862d9b1c002c 100644 (file)
--- a/Documentation/git-svnimport.txt
+++ b/Documentation/git-svnimport.txt
@@ -10,7 +10,7 @@ git-svnimport - Import a SVN repository into git
 SYNOPSIS
 --------
 'git-svnimport' [ -o <branch-for-HEAD> ] [ -h ] [ -v ] [ -d | -D ]
-                       [ -C <GIT_repository> ] [ -i ] [ -u ] [-l limit_nr_changes]
+                       [ -C <GIT_repository> ] [ -i ] [ -u ] [-l limit_rev]
                        [ -b branch_subdir ] [ -t trunk_subdir ] [ -T tag_subdir ]
                        [ -s start_chg ] [ -m ] [ -M regex ]
                        <SVN_repository_URL> [ <path> ]
@@ -71,14 +71,11 @@ When importing incementally, you might need to edit the .git/svn2git file.
        regex. It can be used with -m to also see the default regexes.
        You must escape forward slashes.
 
--l <max_num_changes>::
-       Limit the number of SVN changesets we pull before quitting.
-       This option is necessary because the SVN library has serious memory
-       leaks; the recommended value for nontrivial imports is 100.
+-l <max_rev>::
+       Specify a maximum revision number to pull.
 
-       git-svnimport will still exit with a zero exit code. You can check
-       the size of the file ".git/svn2git" to determine whether to call
-       the importer again.
+       Formerly, this option controlled how many revisions to pull, due to
+       SVN memory leaks. (These have been worked around.)
 
 -v::
        Verbosity: let 'svnimport' report what it is doing.

diff --git a/git-svnimport.perl b/git-svnimport.perl
index 45d77c5bae9c055641c416676c1d783c73fc28bb..65868a91e5f88c41942da89b84f169ed4264615d 100755 (executable)
--- a/git-svnimport.perl
+++ b/git-svnimport.perl
@@ -35,7 +35,7 @@ our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,$opt_b
 sub usage() {
        print STDERR <<END;
 Usage: ${\basename $0}     # fetch/update GIT from SVN
-       [-o branch-for-HEAD] [-h] [-v] [-l max_num_changes]
+       [-o branch-for-HEAD] [-h] [-v] [-l max_rev]
        [-C GIT_repository] [-t tagname] [-T trunkname] [-b branchname]
        [-d|-D] [-i] [-u] [-s start_chg] [-m] [-M regex] [SVN_URL]
 END
@@ -126,8 +126,9 @@ sub file {
 package main;
 use URI;
 
-my $svn = $svn_url;
+our $svn = $svn_url;
 $svn .= "/$svn_dir" if defined $svn_dir;
+my $svn2 = SVNconn->new($svn);
 $svn = SVNconn->new($svn);
 
 my $lwp_ua;
@@ -198,7 +199,7 @@ $ENV{GIT_INDEX_FILE} = $git_index;
 my $maxnum = 0;
 my $last_rev = "";
 my $last_branch;
-my $current_rev = $opt_s-1;
+my $current_rev = $opt_s || 1;
 unless(-d $git_dir) {
        system("git-init-db");
        die "Cannot init the GIT db at $git_tree: $?\n" if $?;
@@ -254,7 +255,7 @@ EOM
                my($num,$branch,$ref) = split;
                $branches{$branch}{$num} = $ref;
                $branches{$branch}{"LAST"} = $ref;
-               $current_rev = $num if $current_rev < $num;
+               $current_rev = $num+1 if $current_rev <= $num;
        }
        close($B);
 }
@@ -708,17 +709,17 @@ sub commit {
        print "DONE: $revision $dest $cid\n" if $opt_v;
 }
 
-my ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
-sub _commit_all {
-       ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
+sub commit_all {
+       # Recursive use of the SVN connection does not work
+       local $svn = $svn2;
+
+       my ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
        my %p;
        while(my($path,$action) = each %$changed_paths) {
                $p{$path} = [ $action->action,$action->copyfrom_path, $action->copyfrom_rev, $path ];
        }
        $changed_paths = \%p;
-}
 
-sub commit_all {
        my %done;
        my @col;
        my $pref;
@@ -734,18 +735,12 @@ sub commit_all {
        }
 }
 
-while(++$current_rev <= $svn->{'maxrev'}) {
-       if (defined $opt_l) {
-               $opt_l--;
-               if ($opt_l < 0) {
-                       last;
-               }
-       }
-       my $pool=SVN::Pool->new;
-       $svn->{'svn'}->get_log("/",$current_rev,$current_rev,1,1,1,\&_commit_all,$pool);
-       $pool->clear;
-       commit_all();
-}
+$opt_l = $svn->{'maxrev'} if not defined $opt_l or $opt_l > $svn->{'maxrev'};
+print "Fetching from $current_rev to $opt_l ...\n" if $opt_v;
+
+my $pool=SVN::Pool->new;
+$svn->{'svn'}->get_log("/",$current_rev,$opt_l,0,1,1,\&commit_all,$pool);
+$pool->clear;
 
 
 unlink($git_index);