Optimize the dependencies list

On a large wiki you can spend a lot of time reading through large lists
of dependencies to see whether files need to be rebuilt (album, with its
one-page-per-photo arrangement, suffers particularly badly from this).

The dependency list is currently a single pagespec, but it's not used like
a normal pagespec - in practice, it's a list of pagespecs joined with the
"or" operator.

Accordingly, change it to be stored as a list of pagespecs. On a wiki
with many tagged photo albums, this reduces the time to refresh after
`touch tags/*.mdwn` from about 31 to 25 seconds.

Getting the benefit of this change on an existing wiki requires a rebuild.

diff --git a/IkiWiki.pm b/IkiWiki.pm
index 43ffb1fd8362802dc19f06074437e1f0a4c87f27..3e94c8a257ce2670089916942c2a542a27fd37a3 100644 (file)
--- a/IkiWiki.pm
+++ b/IkiWiki.pm
@@ -1507,8 +1507,11 @@ sub loadindex () {
                                $links{$page}=$d->{links};
                                $oldlinks{$page}=[@{$d->{links}}];
                        }
                                $links{$page}=$d->{links};
                                $oldlinks{$page}=[@{$d->{links}}];
                        }

-                       if (exists $d->{depends}) {
-                               $depends{$page}=$d->{depends};
+                       if (exists $d->{dependslist}) {
+                               $depends{$page}=$d->{dependslist};
+                       }
+                       elsif (exists $d->{depends}) {
+                               $depends{$page}=[$d->{depends}];

                        }
                        if (exists $d->{state}) {
                                $pagestate{$page}=$d->{state};
                        }
                        if (exists $d->{state}) {
                                $pagestate{$page}=$d->{state};


@@ -1554,7 +1557,8 @@ sub saveindex () {
                };
 
                if (exists $depends{$page}) {
                };
 
                if (exists $depends{$page}) {

-                       $index{page}{$src}{depends} = $depends{$page};
+                       $index{page}{$src}{depends} = join(" or ", @{$depends{$page}});
+                       $index{page}{$src}{dependslist} = $depends{$page};

                }
 
                if (exists $pagestate{$page}) {
                }
 
                if (exists $pagestate{$page}) {


@@ -1724,14 +1728,17 @@ sub rcs_receive () {
 sub add_depends ($$) {
        my $page=shift;
        my $pagespec=shift;
 sub add_depends ($$) {
        my $page=shift;
        my $pagespec=shift;

-       
+

        return unless pagespec_valid($pagespec);
 
        if (! exists $depends{$page}) {
        return unless pagespec_valid($pagespec);
 
        if (! exists $depends{$page}) {

-               $depends{$page}=$pagespec;
+               $depends{$page}=[$pagespec];

        }
        else {
        }
        else {

-               $depends{$page}=pagespec_merge($depends{$page}, $pagespec);
+               foreach my $p (@{$depends{$page}}) {
+                       return 1 if $p eq $pagespec;
+               }
+               push @{$depends{$page}}, $pagespec;

        }
 
        return 1;
        }
 
        return 1;

diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index 578142d2e8eddb5b41dd08ce13a1e65b54ddf1c6..08d484847a5eef33235d33cfdc0d7a39fc5cfbbe 100644 (file)
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -455,20 +455,22 @@ sub refresh () {
                my @changed=(keys %rendered, @del);
 
                # rebuild dependant pages
                my @changed=(keys %rendered, @del);
 
                # rebuild dependant pages

-               foreach my $f (@$files) {
+               F: foreach my $f (@$files) {

                        next if $rendered{$f};
                        my $p=pagename($f);
                        if (exists $depends{$p}) {
                        next if $rendered{$f};
                        my $p=pagename($f);
                        if (exists $depends{$p}) {

-                               # only consider internal files
-                               # if the page explicitly depends on such files
-                               foreach my $file (@changed, $depends{$p}=~/internal\(/ ? @internal : ()) {
-                                       next if $f eq $file;
-                                       my $page=pagename($file);
-                                       if (pagespec_match($page, $depends{$p}, location => $p)) {
-                                               debug(sprintf(gettext("building %s, which depends on %s"), $f, $page));
-                                               render($f);
-                                               $rendered{$f}=1;
-                                               last;
+                               foreach my $d (@{$depends{$p}}) {
+                                       # only consider internal files
+                                       # if the page explicitly depends on such files
+                                       foreach my $file (@changed, $d=~/internal\(/ ? @internal : ()) {
+                                               next if $f eq $file;
+                                               my $page=pagename($file);
+                                               if (pagespec_match($page, $d, location => $p)) {
+                                                       debug(sprintf(gettext("building %s, which depends on %s"), $f, $page));
+                                                       render($f);
+                                                       $rendered{$f}=1;
+                                                       next F;
+                                               }

                                        }
                                }
                        }
                                        }
                                }
                        }

diff --git a/ikiwiki-transition b/ikiwiki-transition
index 398b1a3c8eb989a59258e0a846ca0cb0a5addcb0..60cea3d54895f531fce1864d5e21224aa1a3e732 100755 (executable)
--- a/ikiwiki-transition
+++ b/ikiwiki-transition
@@ -299,7 +299,7 @@ sub oldloadindex {
                        $pagemtime{$page}=$items{mtime}[0];
                        $oldlinks{$page}=[@{$items{link}}];
                        $links{$page}=[@{$items{link}}];
                        $pagemtime{$page}=$items{mtime}[0];
                        $oldlinks{$page}=[@{$items{link}}];
                        $links{$page}=[@{$items{link}}];

-                       $depends{$page}=$items{depends}[0] if exists $items{depends};
+                       $depends{$page}=[$items{depends}[0]] if exists $items{depends};

                        $destsources{$_}=$page foreach @{$items{dest}};
                        $renderedfiles{$page}=[@{$items{dest}}];
                        $pagecase{lc $page}=$page;
                        $destsources{$_}=$page foreach @{$items{dest}};
                        $renderedfiles{$page}=[@{$items{dest}}];
                        $pagecase{lc $page}=$page;

diff --git a/t/index.t b/t/index.t
index e79609902a5856f4a58cc47dec9477df074a0030..107dac9d0522e73d435738d692af1e7a17cfb358 100755 (executable)
--- a/t/index.t
+++ b/t/index.t
@@ -32,9 +32,9 @@ $renderedfiles{"bar.png"}=["bar.png"];
 $links{"Foo"}=["bar.png"];
 $links{"bar"}=["Foo", "new-page"];
 $links{"bar.png"}=[];
 $links{"Foo"}=["bar.png"];
 $links{"bar"}=["Foo", "new-page"];
 $links{"bar.png"}=[];

-$depends{"Foo"}="";
-$depends{"bar"}="foo*";
-$depends{"bar.png"}="";
+$depends{"Foo"}=[];
+$depends{"bar"}=["foo*"];
+$depends{"bar.png"}=[];

 $pagestate{"bar"}{meta}{title}="a page about bar";
 $pagestate{"bar"}{meta}{moo}="mooooo";
 # only loaded plugins save state, so this should not be saved out
 $pagestate{"bar"}{meta}{title}="a page about bar";
 $pagestate{"bar"}{meta}{moo}="mooooo";
 # only loaded plugins save state, so this should not be saved out


@@ -80,9 +80,9 @@ is_deeply(\%links, {
        "bar.png" => [],
 }, "%links loaded correctly");
 is_deeply(\%depends, {
        "bar.png" => [],
 }, "%links loaded correctly");
 is_deeply(\%depends, {

-       Foo => "",
-       bar => "foo*",
-       "bar.png" => "",
+       Foo => [],
+       bar => ["foo*"],
+       "bar.png" => [],

 }, "%depends loaded correctly");
 is_deeply(\%pagestate, {
        bar => {
 }, "%depends loaded correctly");
 is_deeply(\%pagestate, {
        bar => {