From 103e1b321f28fbe61c09012d764944b4b9a3b79f Mon Sep 17 00:00:00 2001 From: david Date: Tue, 20 Nov 2012 22:23:20 +2000 Subject: [PATCH] [PATCH 2/2] test: initial performance testing infrastructure --- 56/0f8df2533c0e795616fb970f1296cbf91bb63c | 330 ++++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 56/0f8df2533c0e795616fb970f1296cbf91bb63c diff --git a/56/0f8df2533c0e795616fb970f1296cbf91bb63c b/56/0f8df2533c0e795616fb970f1296cbf91bb63c new file mode 100644 index 000000000..f4370b896 --- /dev/null +++ b/56/0f8df2533c0e795616fb970f1296cbf91bb63c @@ -0,0 +1,330 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 15991431FC3 + for ; Mon, 19 Nov 2012 18:23:42 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: 0 +X-Spam-Level: +X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none] + autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id 9L18IKnSUDM2 for ; + Mon, 19 Nov 2012 18:23:40 -0800 (PST) +Received: from tesseract.cs.unb.ca (tesseract.cs.unb.ca [131.202.240.238]) + (using TLSv1 with cipher AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 5A0F5431FBC + for ; Mon, 19 Nov 2012 18:23:40 -0800 (PST) +Received: from fctnnbsc30w-156034089108.dhcp-dynamic.fibreop.nb.bellaliant.net + ([156.34.89.108] helo=zancas.localnet) + by tesseract.cs.unb.ca with esmtpsa + (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.72) + (envelope-from ) + id 1TadUw-0004EO-LB; Mon, 19 Nov 2012 22:23:39 -0400 +Received: from bremner by zancas.localnet with local (Exim 4.80) + (envelope-from ) + id 1TadUr-0007SO-37; Mon, 19 Nov 2012 22:23:33 -0400 +From: david@tethera.net +To: notmuch@notmuchmail.org +Subject: [PATCH 2/2] test: initial performance testing infrastructure +Date: Mon, 19 Nov 2012 22:23:20 -0400 +Message-Id: <1353378200-28625-2-git-send-email-david@tethera.net> +X-Mailer: git-send-email 1.7.10.4 +In-Reply-To: <1353378200-28625-1-git-send-email-david@tethera.net> +References: <1353188234-29666-1-git-send-email-david@tethera.net> + <1353378200-28625-1-git-send-email-david@tethera.net> +X-Spam_bar: - +Cc: David Bremner +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Tue, 20 Nov 2012 02:23:42 -0000 + +From: David Bremner + +This is not near as fancy as as the unit tests, on the theory that +the code should typically be crashing when performance tuning. +Nonetheless, there is plenty of room for improvement. Several more of +the pieces of the test infrastructure (e.g. the option parsing) could +be factored out into test/test-lib-common.sh +--- + Makefile | 3 +- + performance-test/.gitignore | 1 + + performance-test/Makefile | 7 +++ + performance-test/Makefile.local | 24 +++++++++ + performance-test/README | 36 +++++++++++++ + performance-test/basic | 15 ++++++ + performance-test/download/.gitignore | 1 + + .../download/notmuch-email-corpus-0.1.tar.gz.asc | 9 ++++ + performance-test/notmuch-perf-test | 25 +++++++++ + performance-test/perf-test-lib.sh | 57 ++++++++++++++++++++ + 10 files changed, 177 insertions(+), 1 deletion(-) + create mode 100644 performance-test/.gitignore + create mode 100644 performance-test/Makefile + create mode 100644 performance-test/Makefile.local + create mode 100644 performance-test/README + create mode 100755 performance-test/basic + create mode 100644 performance-test/download/.gitignore + create mode 100644 performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc + create mode 100755 performance-test/notmuch-perf-test + create mode 100644 performance-test/perf-test-lib.sh + +diff --git a/Makefile b/Makefile +index bb9c316..5decbea 100644 +--- a/Makefile ++++ b/Makefile +@@ -3,7 +3,8 @@ + all: + + # List all subdirectories here. Each contains its own Makefile.local +-subdirs = compat completion emacs lib man parse-time-string util test ++subdirs := compat completion emacs lib man parse-time-string ++subdirs := $(subdirs) performance-test util test + + # We make all targets depend on the Makefiles themselves. + global_deps = Makefile Makefile.config Makefile.local \ +diff --git a/performance-test/.gitignore b/performance-test/.gitignore +new file mode 100644 +index 0000000..53f2697 +--- /dev/null ++++ b/performance-test/.gitignore +@@ -0,0 +1 @@ ++tmp.*/ +diff --git a/performance-test/Makefile b/performance-test/Makefile +new file mode 100644 +index 0000000..de492a7 +--- /dev/null ++++ b/performance-test/Makefile +@@ -0,0 +1,7 @@ ++# See Makefile.local for the list of files to be compiled in this ++# directory. ++all: ++ $(MAKE) -C .. all ++ ++.DEFAULT: ++ $(MAKE) -C .. $@ +diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local +new file mode 100644 +index 0000000..d20457e +--- /dev/null ++++ b/performance-test/Makefile.local +@@ -0,0 +1,24 @@ ++# -*- makefile -*- ++ ++dir := performance-test ++ ++PERFTEST_VERSION := 0.1 ++ ++TGZFILE := $(dir)/download/notmuch-email-corpus-$(PERFTEST_VERSION).tar.gz ++SIGFILE := $(TGZFILE).asc ++TEST_SCRIPT := $(dir)/notmuch-perf-test ++ ++perf-test: setup-perf-test all ++ $(TEST_SCRIPT) $(OPTIONS) ++ ++setup-perf-test: $(TGZFILE) ++ gpg --verify $(SIGFILE) ++ ++$(TGZFILE): ++ @echo ++ @echo Please download ${TGZFILE} ++ @echo See http://notmuchmail.org/corpus for download locations ++ @echo ++ @false ++ ++CLEAN := $(CLEAN) $(dir)/tmp.* +diff --git a/performance-test/README b/performance-test/README +new file mode 100644 +index 0000000..1f98337 +--- /dev/null ++++ b/performance-test/README +@@ -0,0 +1,36 @@ ++Pre-requisites ++-------------- ++ ++In addition to having notmuch, you need: ++ ++- gpg ++- gnu tar ++- gnu time ++ ++Getting set up to run tests: ++---------------------------- ++ ++First, you need to get the corpus. ++ ++It should work to run ++ ++ % cd download ++ % wget http://notmuchmail.org/releases/notmuch-email-corpus-${V}.tar.gz ++ ++Where $V is the current version. ++ ++In case that fails or is too slow, check ++ ++ http://notmuchmail.org/corpus ++ ++for a list of mirrors. ++ ++Running tests ++------------- ++ ++The easiest way to run performance tests is to say "make perf-test", (or ++simply run the notmuch-perf-test script). Either command will run all ++available performance tests. ++ ++Alternately, you can run a specific subset of tests by simply invoking ++one of the executable scripts in this directory, (such as ./basic). +diff --git a/performance-test/basic b/performance-test/basic +new file mode 100755 +index 0000000..9d015ee +--- /dev/null ++++ b/performance-test/basic +@@ -0,0 +1,15 @@ ++#!/bin/bash ++ ++. ./perf-test-lib.sh ++ ++add_email_corpus ++ ++print_header ++ ++time_run 'initial notmuch new' 'notmuch new' ++time_run 'second notmuch new' 'notmuch new' ++time_run 'dump *' 'notmuch dump > tags.out' ++time_run 'restore *' 'notmuch restore < tags.out' ++time_run 'tag * +new_tag' "notmuch tag +new_tag '*'" ++ ++time_done +diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore +new file mode 100644 +index 0000000..335ec95 +--- /dev/null ++++ b/performance-test/download/.gitignore +@@ -0,0 +1 @@ ++*.tar.gz +diff --git a/performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc b/performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc +new file mode 100644 +index 0000000..1041523 +--- /dev/null ++++ b/performance-test/download/notmuch-email-corpus-0.1.tar.gz.asc +@@ -0,0 +1,9 @@ ++-----BEGIN PGP SIGNATURE----- ++Version: GnuPG v1.4.12 (GNU/Linux) ++ ++iJwEAAECAAYFAlCqaPMACgkQTiiN/0Um85mDdQP/c+AErVD2ix+1Jf159EigEpND ++Kzmak2c0Ae2bsfkjZa07GIc9T147lloSg+TKO3PMXTaXHCyYYolgNjZsD8SnP3Hp ++nF2zgNrRlVpaMysqJX005OMbb3cVUrKHQpOCZ764dFyVPCi7ag42HRy/XrODUD3w ++HJsx8ejTFbxfYi9c2+g= ++=4ZIO ++-----END PGP SIGNATURE----- +diff --git a/performance-test/notmuch-perf-test b/performance-test/notmuch-perf-test +new file mode 100755 +index 0000000..1bea345 +--- /dev/null ++++ b/performance-test/notmuch-perf-test +@@ -0,0 +1,25 @@ ++#!/usr/bin/env bash ++ ++# Run tests ++# ++# Copyright (c) 2005 Junio C Hamano ++# ++# Adapted from a Makefile to a shell script by Carl Worth (2010) ++ ++if [ ${BASH_VERSINFO[0]} -lt 4 ]; then ++ echo "Error: The notmuch test suite requires a bash version >= 4.0" ++ echo "due to use of associative arrays within the test suite." ++ echo "Please try again with a newer bash (or help us fix the" ++ echo "test suite to be more portable). Thanks." ++ exit 1 ++fi ++ ++cd $(dirname "$0") ++ ++TESTS=" ++ basic ++" ++ ++for test in $TESTS; do ++ ./$test "$@" ++done +diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh +new file mode 100644 +index 0000000..8e6004f +--- /dev/null ++++ b/performance-test/perf-test-lib.sh +@@ -0,0 +1,57 @@ ++ ++. ../test/test-lib-common.sh ++ ++set -e ++ ++if ! test -x ../notmuch ++then ++ echo >&2 'You do not seem to have built notmuch yet.' ++ exit 1 ++fi ++ ++add_email_corpus () ++{ ++ rm -rf ${MAIL_DIR} ++ ++ arg="" ++ case "$1" in ++ --small) ++ arg="/enron/bailey-s" ++ ;; ++ --medium) ++ arg="/notmuch-archive" ++ ;; ++ esac ++ ++ printf "Unpacking corpus\n" ++ tar --checkpoint=.5000 --extract --strip-components=1 \ ++ --directory ${TMP_DIRECTORY} \ ++ --file ../download/notmuch-email-corpus-0.1.tar.gz \ ++ notmuch-email-corpus/mail"$arg" ++ ++ printf "\n" ++} ++ ++print_header () { ++ printf " Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn\tOut\n" ++} ++ ++time_run () { ++ printf "%-22s" "$1" ++ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi ++ if ! eval >&3 "/usr/bin/time -f '%e\t%U\t%S\t%M\t%I\t%O' $2" ; then ++ test_failure=$(($test_failure + 1)) ++ fi ++} ++ ++time_done () { ++ if [ "$test_failure" = "0" ]; then ++ rm -rf "$remove_tmp" ++ exit 0 ++ else ++ exit 1 ++ fi ++} ++ ++cd -P "$test" || error "Cannot setup test environment" ++test_failure=0 +-- +1.7.10.4 + -- 2.26.2