--- /dev/null
+Return-Path: <bremner@tethera.net>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 988EB429E26\r
+ for <notmuch@notmuchmail.org>; Sun, 16 Dec 2012 10:23:59 -0800 (PST)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: 0\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none]\r
+ autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id ftMJZPQfbZnT for <notmuch@notmuchmail.org>;\r
+ Sun, 16 Dec 2012 10:23:57 -0800 (PST)\r
+Received: from tesseract.cs.unb.ca (tesseract.cs.unb.ca [131.202.240.238])\r
+ (using TLSv1 with cipher AES256-SHA (256/256 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id AF86B429E2F\r
+ for <notmuch@notmuchmail.org>; Sun, 16 Dec 2012 10:23:56 -0800 (PST)\r
+Received: from fctnnbsc30w-142167090129.dhcp-dynamic.fibreop.nb.bellaliant.net\r
+ ([142.167.90.129] helo=zancas.localnet)\r
+ by tesseract.cs.unb.ca with esmtpsa\r
+ (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.72)\r
+ (envelope-from <bremner@tethera.net>)\r
+ id 1TkIsV-0007mx-GU; Sun, 16 Dec 2012 14:23:56 -0400\r
+Received: from bremner by zancas.localnet with local (Exim 4.80)\r
+ (envelope-from <bremner@tethera.net>)\r
+ id 1TkIsQ-0003cY-0U; Sun, 16 Dec 2012 14:23:50 -0400\r
+From: david@tethera.net\r
+To: notmuch@notmuchmail.org\r
+Subject: [PATCH 3/4] perf-test: initial version of memory test infrastructure.\r
+Date: Sun, 16 Dec 2012 14:23:27 -0400\r
+Message-Id: <1355682208-13729-4-git-send-email-david@tethera.net>\r
+X-Mailer: git-send-email 1.7.10.4\r
+In-Reply-To: <1355682208-13729-1-git-send-email-david@tethera.net>\r
+References: <1355682208-13729-1-git-send-email-david@tethera.net>\r
+X-Spam_bar: -\r
+Cc: David Bremner <bremner@debian.org>\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sun, 16 Dec 2012 18:24:00 -0000\r
+\r
+From: David Bremner <bremner@debian.org>\r
+\r
+The idea is run some code under valgrind --leak-check=full and report\r
+a summary, leaving the user to peruse the log file if they want.\r
+\r
+We go to some lengths to preserve the log files from accidental\r
+overwriting; the full corpus takes about 3 hours to run under valgrind\r
+on my machine.\r
+\r
+The naming of the log directories is probably overkill; I find it nice\r
+to have them sequenced by time. Arguably the mktemp is then overkill,\r
+but I know people will be nervous if it looks like timestamps are\r
+being used for uniqueness.\r
+\r
+One new test is included, to check notmuch new for memory leaks.\r
+---\r
+ performance-test/.gitignore | 1 +\r
+ performance-test/M00-new | 14 +++++++++\r
+ performance-test/Makefile.local | 17 +++++++++--\r
+ performance-test/README | 57 +++++++++++++++++++++++++------------\r
+ performance-test/perf-test-lib.sh | 55 ++++++++++++++++++++++++++++-------\r
+ 5 files changed, 113 insertions(+), 31 deletions(-)\r
+ create mode 100755 performance-test/M00-new\r
+\r
+diff --git a/performance-test/.gitignore b/performance-test/.gitignore\r
+index 6421a9a..f3f9be4 100644\r
+--- a/performance-test/.gitignore\r
++++ b/performance-test/.gitignore\r
+@@ -1,3 +1,4 @@\r
+ tmp.*/\r
++log.*/\r
+ corpus/\r
+ notmuch.cache.*/\r
+diff --git a/performance-test/M00-new b/performance-test/M00-new\r
+new file mode 100755\r
+index 0000000..733e9b0\r
+--- /dev/null\r
++++ b/performance-test/M00-new\r
+@@ -0,0 +1,14 @@\r
++#!/bin/bash\r
++\r
++test_description='notmuch new'\r
++\r
++. ./perf-test-lib.sh\r
++\r
++# ensure initial 'notmuch new' is run by memory_start\r
++uncache_database\r
++\r
++memory_start\r
++\r
++memory_run "notmuch new" "notmuch new"\r
++\r
++memory_done\r
+diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local\r
+index 57beb44..357d800 100644\r
+--- a/performance-test/Makefile.local\r
++++ b/performance-test/Makefile.local\r
+@@ -4,14 +4,25 @@ dir := performance-test\r
+ \r
+ include $(dir)/version.sh\r
+ \r
++# these two are just make sure dir is expanded at the right time.\r
++TIME_TEST_SCRIPT := ${dir}/notmuch-time-test\r
++MEMORY_TEST_SCRIPT := ${dir}/notmuch-memory-test\r
++\r
+ CORPUS_NAME := notmuch-email-corpus-$(PERFTEST_VERSION).tar.xz\r
+ TXZFILE := ${dir}/download/${CORPUS_NAME}\r
+ SIGFILE := ${TXZFILE}.asc\r
+-TEST_SCRIPT := ${dir}/notmuch-perf-test\r
+ DEFAULT_URL := http://notmuchmail.org/releases/${CORPUS_NAME}\r
+ \r
++perf-test: time-test memory-test\r
++\r
+ time-test: setup-perf-test all\r
+- $(TEST_SCRIPT) $(OPTIONS)\r
++ @echo\r
++ $(TIME_TEST_SCRIPT) $(TEST_OPTIONS)\r
++\r
++memory-test: setup-perf-test all\r
++ @echo\r
++ $(MEMORY_TEST_SCRIPT) $(TEST_OPTIONS)\r
++\r
+ \r
+ .PHONY: download-corpus setup-perf-test\r
+ \r
+@@ -29,4 +40,4 @@ $(TXZFILE):\r
+ download-corpus:\r
+ wget -O ${TXZFILE} ${DEFAULT_URL}\r
+ \r
+-CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/corpus $(dir)/notmuch.cache.*\r
++CLEAN := $(CLEAN) $(dir)/tmp.* $(dir)/log.* $(dir)/corpus $(dir)/notmuch.cache.*\r
+diff --git a/performance-test/README b/performance-test/README\r
+index d1fb6de..7eaf5f7 100644\r
+--- a/performance-test/README\r
++++ b/performance-test/README\r
+@@ -1,3 +1,10 @@\r
++Performance Tests\r
++-----------------\r
++\r
++This directory contains two kinds of performance tests, time tests,\r
++and memory tests. The former use gnu time, and the latter use\r
++valgrind.\r
++\r
+ Pre-requisites\r
+ --------------\r
+ \r
+@@ -5,9 +12,10 @@ In addition to having notmuch, you need:\r
+ \r
+ - gpg\r
+ - gnu tar\r
+-- gnu time\r
++- gnu time (for the time tests).\r
+ - xz. Some speedup can be gotten by installing "pixz", but this is\r
+ probably only worthwhile if you are debugging the tests.\r
++- valgrind (for the memory tests)\r
+ \r
+ Getting set up to run tests:\r
+ ----------------------------\r
+@@ -36,34 +44,47 @@ for a list of mirrors.\r
+ Running tests\r
+ -------------\r
+ \r
+-The easiest way to run performance tests is to say "make time-test", (or\r
+-simply run the notmuch-time-test script). Either command will run all\r
+-available performance tests.\r
+-\r
+-Alternately, you can run a specific subset of tests by simply invoking\r
+-one of the executable scripts in this directory, (such as ./basic).\r
+-Each test script supports the following arguments\r
++The easiest way to run performance tests is to say "make perf-test".\r
++This will run all time and memory tests. Be aware that the memory\r
++tests are quite time consuming when run on the full corpus, and that\r
++depending on your interests it may be more sensible to run "make\r
++time-test" or "make memory-test". You can also invoke one of the\r
++scripts notmuch-time-test or notmuch-memory-test or run a more\r
++specific subset of tests by simply invoking one of the executable\r
++scripts in this directory, (such as ./T00-new). Each test script\r
++supports the following arguments\r
+ \r
+ --small / --medium / --large Choose corpus size.\r
+ --debug Enable debugging. In particular don't delete\r
+ temporary directories.\r
+ \r
++When using the make targets, you can pass arguments to all test\r
++scripts by defining the make variable TEST_OPTIONS.\r
++\r
+ Writing tests\r
+ -------------\r
+ \r
+-Have a look at "T01-dump-restore" for an example. Sourcing\r
+-"perf-test-lib.sh" is mandatory. Utility functions include\r
++Have a look at "T01-dump-restore" for an example time test and and\r
++"M00-new" for an example memory tests. In both cases sourcing\r
++"perf-test-lib.sh" is mandatory.\r
+ \r
+-- 'add_email_corpus' unpacks a set of messages and adds them to the database.\r
+-- 'cache_database': makes a snapshot of the current database\r
+-- 'uncache_database': forces the next 'add_email_corpus' to rebuild the\r
+- database.\r
+-- 'time_start' unpacks the mail corpus and calls notmuch new if it\r
++Basics:\r
++\r
++- '(time|memory)_start' unpacks the mail corpus and calls notmuch new if it\r
+ cannot find a cache of the appropriate corpus.\r
+-- 'time_done' does the cleanup; comment it out or pass --debug to the\r
++- '(time|memory)_run' runs the command under time or valgrind. Currently\r
++ "memory_run" does not support i/o redirection in the command.\r
++- '(time|memory)_done' does the cleanup; comment it out or pass --debug to the\r
+ script to leave the temporary files around.\r
+ \r
++Utility functions include\r
++\r
++- 'add_email_corpus' unpacks a set of messages and tags\r
++- 'cache_database': makes a snapshot of the current database\r
++- 'uncache_database': forces the next '(time|memory)_start' to rebuild the\r
++ database.\r
++\r
+ Scripts are run in the order specified in notmuch-perf-test. In the\r
+ future this order might be chosen automatically so please follow the\r
+-convention of starting the name with 'T' followed by two digits to\r
+-specify the order.\r
++convention of starting the name with 'T' or 'M' followed by two digits\r
++to specify the order.\r
+diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh\r
+index 3a7ef2b..79eb2c5 100644\r
+--- a/performance-test/perf-test-lib.sh\r
++++ b/performance-test/perf-test-lib.sh\r
+@@ -92,20 +92,52 @@ add_email_corpus ()\r
+ \r
+ }\r
+ \r
+-time_start () {\r
+-\r
+- add_email_corpus\r
+-\r
+- print_header\r
++notmuch_new_with_cache () {\r
+ \r
+ if [ -d $DB_CACHE_DIR ]; then\r
+ cp -r $DB_CACHE_DIR ${MAIL_DIR}/.notmuch\r
+ else\r
+- time_run 'Initial notmuch new' "notmuch new"\r
++ "$1" 'Initial notmuch new' "notmuch new"\r
+ cache_database\r
+ fi\r
+ }\r
+ \r
++time_start () {\r
++\r
++ add_email_corpus\r
++\r
++ print_header\r
++\r
++ notmuch_new_with_cache time_run\r
++}\r
++\r
++memory_start () {\r
++\r
++ add_email_corpus\r
++\r
++ _timestamp=$(printf "%x" $(date +"%s"))\r
++ log_dir=$(mktemp -d "${TEST_DIRECTORY}/log.$(basename $0)-$corpus_size-${_timestamp}-XXXXXX")\r
++\r
++ notmuch_new_with_cache memory_run\r
++}\r
++\r
++memory_run () {\r
++ test_count=$(($test_count+1))\r
++\r
++ log_file=$log_dir/$test_count.log\r
++\r
++ printf "[ %d ]\t%s\n" $test_count "$1"\r
++\r
++ valgrind --leak-check=full --log-file="$log_file" $2\r
++\r
++ awk '/LEAK SUMMARY/,/suppressed/ { sub(/^==[0-9]*==/," "); print }' "$log_file"\r
++ echo\r
++}\r
++\r
++memory_done () {\r
++ time_done\r
++}\r
++\r
+ cache_database () {\r
+ if [ -d $MAIL_DIR/.notmuch ]; then\r
+ cp -r $MAIL_DIR/.notmuch $DB_CACHE_DIR\r
+@@ -119,12 +151,13 @@ uncache_database () {\r
+ }\r
+ \r
+ print_header () {\r
+- printf "[v%4s %6s] Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n" \\r
+- ${PERFTEST_VERSION} ${corpus_size}\r
++ printf "\t\t\tWall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn/Out(512B)\n"\r
++\r
+ }\r
+ \r
+ time_run () {\r
+ printf " %-22s" "$1"\r
++ test_count=$(($test_count+1))\r
+ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi\r
+ if ! eval >&3 "/usr/bin/time -f '%e\t%U\t%S\t%M\t%I/%O' $2" ; then\r
+ test_failure=$(($test_failure + 1))\r
+@@ -144,6 +177,8 @@ time_done () {\r
+ \r
+ cd -P "$test" || error "Cannot setup test environment"\r
+ test_failure=0\r
++test_count=0\r
+ \r
+-echo\r
+-echo $(basename "$0"): "Testing ${test_description:-notmuch performance}"\r
++printf "\n%-55s [%s %s]\n" \\r
++ "$(basename "$0"): Testing ${test_description:-notmuch performance}" \\r
++ "${PERFTEST_VERSION}" "${corpus_size}"\r
+-- \r
+1.7.10.4\r
+\r
projects / notmuch-archives.git / commitdiff