From fe4f176f725b069ac74561600bba982c2d9ad607 Mon Sep 17 00:00:00 2001 From: Simon McVittie Date: Thu, 18 Jun 2009 15:54:53 +0100 Subject: [PATCH] Optimize the dependencies list On a large wiki you can spend a lot of time reading through large lists of dependencies to see whether files need to be rebuilt (album, with its one-page-per-photo arrangement, suffers particularly badly from this). The dependency list is currently a single pagespec, but it's not used like a normal pagespec - in practice, it's a list of pagespecs joined with the "or" operator. Accordingly, change it to be stored as a list of pagespecs. On a wiki with many tagged photo albums, this reduces the time to refresh after `touch tags/*.mdwn` from about 31 to 25 seconds. Getting the benefit of this change on an existing wiki requires a rebuild. --- IkiWiki.pm | 19 +++++++++++++------ IkiWiki/Render.pm | 24 +++++++++++++----------- ikiwiki-transition | 2 +- t/index.t | 12 ++++++------ 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/IkiWiki.pm b/IkiWiki.pm index 43ffb1fd8..3e94c8a25 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -1507,8 +1507,11 @@ sub loadindex () { $links{$page}=$d->{links}; $oldlinks{$page}=[@{$d->{links}}]; } - if (exists $d->{depends}) { - $depends{$page}=$d->{depends}; + if (exists $d->{dependslist}) { + $depends{$page}=$d->{dependslist}; + } + elsif (exists $d->{depends}) { + $depends{$page}=[$d->{depends}]; } if (exists $d->{state}) { $pagestate{$page}=$d->{state}; @@ -1554,7 +1557,8 @@ sub saveindex () { }; if (exists $depends{$page}) { - $index{page}{$src}{depends} = $depends{$page}; + $index{page}{$src}{depends} = join(" or ", @{$depends{$page}}); + $index{page}{$src}{dependslist} = $depends{$page}; } if (exists $pagestate{$page}) { @@ -1724,14 +1728,17 @@ sub rcs_receive () { sub add_depends ($$) { my $page=shift; my $pagespec=shift; - + return unless pagespec_valid($pagespec); if (! exists $depends{$page}) { - $depends{$page}=$pagespec; + $depends{$page}=[$pagespec]; } else { - $depends{$page}=pagespec_merge($depends{$page}, $pagespec); + foreach my $p (@{$depends{$page}}) { + return 1 if $p eq $pagespec; + } + push @{$depends{$page}}, $pagespec; } return 1; diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm index 578142d2e..08d484847 100644 --- a/IkiWiki/Render.pm +++ b/IkiWiki/Render.pm @@ -455,20 +455,22 @@ sub refresh () { my @changed=(keys %rendered, @del); # rebuild dependant pages - foreach my $f (@$files) { + F: foreach my $f (@$files) { next if $rendered{$f}; my $p=pagename($f); if (exists $depends{$p}) { - # only consider internal files - # if the page explicitly depends on such files - foreach my $file (@changed, $depends{$p}=~/internal\(/ ? @internal : ()) { - next if $f eq $file; - my $page=pagename($file); - if (pagespec_match($page, $depends{$p}, location => $p)) { - debug(sprintf(gettext("building %s, which depends on %s"), $f, $page)); - render($f); - $rendered{$f}=1; - last; + foreach my $d (@{$depends{$p}}) { + # only consider internal files + # if the page explicitly depends on such files + foreach my $file (@changed, $d=~/internal\(/ ? @internal : ()) { + next if $f eq $file; + my $page=pagename($file); + if (pagespec_match($page, $d, location => $p)) { + debug(sprintf(gettext("building %s, which depends on %s"), $f, $page)); + render($f); + $rendered{$f}=1; + next F; + } } } } diff --git a/ikiwiki-transition b/ikiwiki-transition index 398b1a3c8..60cea3d54 100755 --- a/ikiwiki-transition +++ b/ikiwiki-transition @@ -299,7 +299,7 @@ sub oldloadindex { $pagemtime{$page}=$items{mtime}[0]; $oldlinks{$page}=[@{$items{link}}]; $links{$page}=[@{$items{link}}]; - $depends{$page}=$items{depends}[0] if exists $items{depends}; + $depends{$page}=[$items{depends}[0]] if exists $items{depends}; $destsources{$_}=$page foreach @{$items{dest}}; $renderedfiles{$page}=[@{$items{dest}}]; $pagecase{lc $page}=$page; diff --git a/t/index.t b/t/index.t index e79609902..107dac9d0 100755 --- a/t/index.t +++ b/t/index.t @@ -32,9 +32,9 @@ $renderedfiles{"bar.png"}=["bar.png"]; $links{"Foo"}=["bar.png"]; $links{"bar"}=["Foo", "new-page"]; $links{"bar.png"}=[]; -$depends{"Foo"}=""; -$depends{"bar"}="foo*"; -$depends{"bar.png"}=""; +$depends{"Foo"}=[]; +$depends{"bar"}=["foo*"]; +$depends{"bar.png"}=[]; $pagestate{"bar"}{meta}{title}="a page about bar"; $pagestate{"bar"}{meta}{moo}="mooooo"; # only loaded plugins save state, so this should not be saved out @@ -80,9 +80,9 @@ is_deeply(\%links, { "bar.png" => [], }, "%links loaded correctly"); is_deeply(\%depends, { - Foo => "", - bar => "foo*", - "bar.png" => "", + Foo => [], + bar => ["foo*"], + "bar.png" => [], }, "%depends loaded correctly"); is_deeply(\%pagestate, { bar => { -- 2.26.2