From: david Date: Sun, 25 Nov 2012 15:02:24 +0000 (+2000) Subject: [Patch v4 2/2] test: initial performance testing infrastructure X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=4c336efb8a6bce799bb29f1f8da4108d5b1dd5be;p=notmuch-archives.git [Patch v4 2/2] test: initial performance testing infrastructure --- diff --git a/7e/e82aee20586547f9787b1b7cdc09c7d1d69c60 b/7e/e82aee20586547f9787b1b7cdc09c7d1d69c60 new file mode 100644 index 000000000..65fdf2f9c --- /dev/null +++ b/7e/e82aee20586547f9787b1b7cdc09c7d1d69c60 @@ -0,0 +1,371 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 21329431FB6 + for ; Sun, 25 Nov 2012 07:02:47 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: 0 +X-Spam-Level: +X-Spam-Status: No, score=0 tagged_above=-999 required=5 tests=[none] + autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id zJXOu7xviP0c for ; + Sun, 25 Nov 2012 07:02:43 -0800 (PST) +Received: from tesseract.cs.unb.ca (tesseract.cs.unb.ca [131.202.240.238]) + (using TLSv1 with cipher AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 06627431FC4 + for ; Sun, 25 Nov 2012 07:02:41 -0800 (PST) +Received: from fctnnbsc30w-156034089108.dhcp-dynamic.fibreop.nb.bellaliant.net + ([156.34.89.108] helo=zancas.localnet) + by tesseract.cs.unb.ca with esmtpsa + (TLS1.0:DHE_RSA_AES_128_CBC_SHA1:16) (Exim 4.72) + (envelope-from ) + id 1TcdjD-0007HR-Du; Sun, 25 Nov 2012 11:02:40 -0400 +Received: from bremner by zancas.localnet with local (Exim 4.80) + (envelope-from ) + id 1Tcdj7-00034t-UQ; Sun, 25 Nov 2012 11:02:33 -0400 +From: david@tethera.net +To: notmuch@notmuchmail.org +Subject: [Patch v4 2/2] test: initial performance testing infrastructure +Date: Sun, 25 Nov 2012 11:02:24 -0400 +Message-Id: <1353855745-11697-3-git-send-email-david@tethera.net> +X-Mailer: git-send-email 1.7.10.4 +In-Reply-To: <1353855745-11697-1-git-send-email-david@tethera.net> +References: <1353855745-11697-1-git-send-email-david@tethera.net> +X-Spam_bar: - +Cc: David Bremner +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sun, 25 Nov 2012 15:02:47 -0000 + +From: David Bremner + +This is not near as fancy as as the unit tests, on the theory that +the code should typically be crashing when performance tuning. +Nonetheless, there is plenty of room for improvement. Several more of +the pieces of the test infrastructure (e.g. the option parsing) could +be factored out into test/test-lib-common.sh +--- + Makefile | 3 +- + performance-test/.gitignore | 1 + + performance-test/Makefile | 7 +++ + performance-test/Makefile.local | 32 ++++++++++ + performance-test/README | 50 +++++++++++++++ + performance-test/basic | 15 +++++ + performance-test/download/.gitignore | 2 + + .../download/notmuch-email-corpus-0.2.tar.xz.asc | 9 +++ + performance-test/notmuch-perf-test | 25 ++++++++ + performance-test/perf-test-lib.sh | 65 ++++++++++++++++++++ + performance-test/version.sh | 3 + + 11 files changed, 211 insertions(+), 1 deletion(-) + create mode 100644 performance-test/.gitignore + create mode 100644 performance-test/Makefile + create mode 100644 performance-test/Makefile.local + create mode 100644 performance-test/README + create mode 100755 performance-test/basic + create mode 100644 performance-test/download/.gitignore + create mode 100644 performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc + create mode 100755 performance-test/notmuch-perf-test + create mode 100644 performance-test/perf-test-lib.sh + create mode 100644 performance-test/version.sh + +diff --git a/Makefile b/Makefile +index bb9c316..5decbea 100644 +--- a/Makefile ++++ b/Makefile +@@ -3,7 +3,8 @@ + all: + + # List all subdirectories here. Each contains its own Makefile.local +-subdirs = compat completion emacs lib man parse-time-string util test ++subdirs := compat completion emacs lib man parse-time-string ++subdirs := $(subdirs) performance-test util test + + # We make all targets depend on the Makefiles themselves. + global_deps = Makefile Makefile.config Makefile.local \ +diff --git a/performance-test/.gitignore b/performance-test/.gitignore +new file mode 100644 +index 0000000..53f2697 +--- /dev/null ++++ b/performance-test/.gitignore +@@ -0,0 +1 @@ ++tmp.*/ +diff --git a/performance-test/Makefile b/performance-test/Makefile +new file mode 100644 +index 0000000..de492a7 +--- /dev/null ++++ b/performance-test/Makefile +@@ -0,0 +1,7 @@ ++# See Makefile.local for the list of files to be compiled in this ++# directory. ++all: ++ $(MAKE) -C .. all ++ ++.DEFAULT: ++ $(MAKE) -C .. $@ +diff --git a/performance-test/Makefile.local b/performance-test/Makefile.local +new file mode 100644 +index 0000000..1114ec1 +--- /dev/null ++++ b/performance-test/Makefile.local +@@ -0,0 +1,32 @@ ++# -*- makefile -*- ++ ++dir := performance-test ++ ++include $(dir)/version.sh ++ ++CORPUS_NAME := notmuch-email-corpus-$(PERFTEST_VERSION).tar.xz ++TXZFILE := ${dir}/download/${CORPUS_NAME} ++SIGFILE := ${TXZFILE}.asc ++TEST_SCRIPT := ${dir}/notmuch-perf-test ++DEFAULT_URL := http://notmuchmail.org/releases/${CORPUS_NAME} ++ ++perf-test: setup-perf-test all ++ $(TEST_SCRIPT) $(OPTIONS) ++ ++.PHONY: download-corpus setup-perf-test ++ ++# Note that this intentionally does not depend on download-corpus. ++setup-perf-test: $(TXZFILE) ++ gpg --verify $(SIGFILE) ++ ++$(TXZFILE): ++ @printf "\nPlease download ${TXZFILE}.\n\n" ++ @printf "\t%% make download-corpus\n\n" ++ @echo or see http://notmuchmail.org/corpus for download locations ++ @echo ++ @false ++ ++download-corpus: ++ wget -O ${TXZFILE} ${DEFAULT_URL} ++ ++CLEAN := $(CLEAN) $(dir)/tmp.* +diff --git a/performance-test/README b/performance-test/README +new file mode 100644 +index 0000000..239d2fb +--- /dev/null ++++ b/performance-test/README +@@ -0,0 +1,50 @@ ++Pre-requisites ++-------------- ++ ++In addition to having notmuch, you need: ++ ++- gpg ++- gnu tar ++- gnu time ++- xz. Some speedup can be gotten by installing "pixz", but this is ++ probably only worthwhile if you are debugging the tests. ++ ++Getting set up to run tests: ++---------------------------- ++ ++First, you need to get the corpus. ++ ++It should work to run ++ ++ % make download-corpus ++ ++In case that fails or is too slow, check ++ ++ http://notmuchmail.org/corpus ++ ++for a list of mirrors. ++ ++Running tests ++------------- ++ ++The easiest way to run performance tests is to say "make perf-test", (or ++simply run the notmuch-perf-test script). Either command will run all ++available performance tests. ++ ++Alternately, you can run a specific subset of tests by simply invoking ++one of the executable scripts in this directory, (such as ./basic). ++ ++ ++Writing tests ++------------- ++ ++Have a look at "basic" for an example. ++ ++add_email_corpus takes arguments "--small" and "--medium" for when you ++want smaller corpuses to check. ++ ++time_done does the cleanup; comment it out or define "$debug" to leave ++the temporary files around. ++ ++Currently there is no option processing (e.g. --debug) in the ++performance tests. +diff --git a/performance-test/basic b/performance-test/basic +new file mode 100755 +index 0000000..9d015ee +--- /dev/null ++++ b/performance-test/basic +@@ -0,0 +1,15 @@ ++#!/bin/bash ++ ++. ./perf-test-lib.sh ++ ++add_email_corpus ++ ++print_header ++ ++time_run 'initial notmuch new' 'notmuch new' ++time_run 'second notmuch new' 'notmuch new' ++time_run 'dump *' 'notmuch dump > tags.out' ++time_run 'restore *' 'notmuch restore < tags.out' ++time_run 'tag * +new_tag' "notmuch tag +new_tag '*'" ++ ++time_done +diff --git a/performance-test/download/.gitignore b/performance-test/download/.gitignore +new file mode 100644 +index 0000000..7b09234 +--- /dev/null ++++ b/performance-test/download/.gitignore +@@ -0,0 +1,2 @@ ++*.tar.gz ++*.tar.xz +diff --git a/performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc b/performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc +new file mode 100644 +index 0000000..c8b4b3d +--- /dev/null ++++ b/performance-test/download/notmuch-email-corpus-0.2.tar.xz.asc +@@ -0,0 +1,9 @@ ++-----BEGIN PGP SIGNATURE----- ++Version: GnuPG v1.4.12 (GNU/Linux) ++ ++iJwEAAECAAYFAlCsvx0ACgkQTiiN/0Um85kZAwP9GgOQ22jK8mr5X4pT/mB8EjSH ++QbndlxxbRrP0ChTqjBQoD3IsTHjNL7W572BfXb/MNo94R/iIQ7yTHCDVNuwBhvKd ++7qgIuW2FUS1uTfJRP5KBNf8JPuin+6wqGe8/+y/iOs+XJSdiYg1ElS49Ntnpg0yl ++btImgEcxTxQ2qfzDS1g= ++=iuZR ++-----END PGP SIGNATURE----- +diff --git a/performance-test/notmuch-perf-test b/performance-test/notmuch-perf-test +new file mode 100755 +index 0000000..1bea345 +--- /dev/null ++++ b/performance-test/notmuch-perf-test +@@ -0,0 +1,25 @@ ++#!/usr/bin/env bash ++ ++# Run tests ++# ++# Copyright (c) 2005 Junio C Hamano ++# ++# Adapted from a Makefile to a shell script by Carl Worth (2010) ++ ++if [ ${BASH_VERSINFO[0]} -lt 4 ]; then ++ echo "Error: The notmuch test suite requires a bash version >= 4.0" ++ echo "due to use of associative arrays within the test suite." ++ echo "Please try again with a newer bash (or help us fix the" ++ echo "test suite to be more portable). Thanks." ++ exit 1 ++fi ++ ++cd $(dirname "$0") ++ ++TESTS=" ++ basic ++" ++ ++for test in $TESTS; do ++ ./$test "$@" ++done +diff --git a/performance-test/perf-test-lib.sh b/performance-test/perf-test-lib.sh +new file mode 100644 +index 0000000..11ccc77 +--- /dev/null ++++ b/performance-test/perf-test-lib.sh +@@ -0,0 +1,65 @@ ++. ./version.sh ++ ++. ../test/test-lib-common.sh ++ ++set -e ++ ++if ! test -x ../notmuch ++then ++ echo >&2 'You do not seem to have built notmuch yet.' ++ exit 1 ++fi ++ ++add_email_corpus () ++{ ++ rm -rf ${MAIL_DIR} ++ ++ arg="" ++ case "$1" in ++ --small) ++ arg="/enron/bailey-s" ++ ;; ++ --medium) ++ arg="/notmuch-archive" ++ ;; ++ esac ++ ++ if command -v pixz > /dev/null; then ++ XZ=pixz ++ else ++ XZ=xz ++ fi ++ ++ printf "Unpacking corpus\n" ++ tar --checkpoint=.5000 --extract --strip-components=1 \ ++ --directory ${TMP_DIRECTORY} \ ++ --use-compress-program ${XZ} \ ++ --file ../download/notmuch-email-corpus-${PERFTEST_VERSION}.tar.xz \ ++ notmuch-email-corpus/mail"$arg" ++ ++ printf "\n" ++} ++ ++print_header () { ++ printf " Wall(s)\tUsr(s)\tSys(s)\tRes(K)\tIn\tOut\n" ++} ++ ++time_run () { ++ printf "%-22s" "$1" ++ if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi ++ if ! eval >&3 "/usr/bin/time -f '%e\t%U\t%S\t%M\t%I\t%O' $2" ; then ++ test_failure=$(($test_failure + 1)) ++ fi ++} ++ ++time_done () { ++ if [ "$test_failure" = "0" ]; then ++ rm -rf "$remove_tmp" ++ exit 0 ++ else ++ exit 1 ++ fi ++} ++ ++cd -P "$test" || error "Cannot setup test environment" ++test_failure=0 +diff --git a/performance-test/version.sh b/performance-test/version.sh +new file mode 100644 +index 0000000..d9270b1 +--- /dev/null ++++ b/performance-test/version.sh +@@ -0,0 +1,3 @@ ++# this should be both a valid Makefile fragment and valid POSIX(ish) shell. ++ ++PERFTEST_VERSION=0.2 +-- +1.7.10.4 +