From 3519069017c00ce7296df2453b56da6f641c65af Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sat, 4 Oct 2014 12:58:03 +2000 Subject: [PATCH] [PATCH] test: Port atomicity test to Python --- 01/316ad41c5f139cd451eb724884830106815bbf | 247 ++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 01/316ad41c5f139cd451eb724884830106815bbf diff --git a/01/316ad41c5f139cd451eb724884830106815bbf b/01/316ad41c5f139cd451eb724884830106815bbf new file mode 100644 index 000000000..337740180 --- /dev/null +++ b/01/316ad41c5f139cd451eb724884830106815bbf @@ -0,0 +1,247 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 15AAB431FBC + for ; Fri, 3 Oct 2014 09:58:36 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -2.3 +X-Spam-Level: +X-Spam-Status: No, score=-2.3 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_MED=-2.3] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id QmaXSL60CxpO for ; + Fri, 3 Oct 2014 09:58:29 -0700 (PDT) +Received: from dmz-mailsec-scanner-2.mit.edu (dmz-mailsec-scanner-2.mit.edu + [18.9.25.13]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 59798431FB6 + for ; Fri, 3 Oct 2014 09:58:29 -0700 (PDT) +X-AuditID: 1209190d-f79c06d000006f95-80-542ed5b40568 +Received: from mailhub-auth-1.mit.edu ( [18.9.21.35]) + (using TLS with cipher AES256-SHA (256/256 bits)) + (Client did not present a certificate) + by dmz-mailsec-scanner-2.mit.edu (Symantec Messaging Gateway) with SMTP + id 7E.EA.28565.4B5DE245; Fri, 3 Oct 2014 12:58:28 -0400 (EDT) +Received: from outgoing.mit.edu (outgoing-auth-1.mit.edu [18.9.28.11]) + by mailhub-auth-1.mit.edu (8.13.8/8.9.2) with ESMTP id s93GwQhN028669; + Fri, 3 Oct 2014 12:58:26 -0400 +Received: from drake.dyndns.org + (static-155-212-141-65.mas.onecommunications.net [155.212.141.65]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.8/8.12.4) with ESMTP id s93GwEBP019243 + (version=TLSv1/SSLv3 cipher=AES128-SHA bits=128 verify=NOT); + Fri, 3 Oct 2014 12:58:24 -0400 +Received: from amthrax by drake.dyndns.org with local (Exim 4.84) + (envelope-from ) + id 1Xa6BJ-00022m-AR; Fri, 03 Oct 2014 12:58:13 -0400 +From: Austin Clements +To: notmuch@notmuchmail.org +Subject: [PATCH] test: Port atomicity test to Python +Date: Fri, 3 Oct 2014 12:58:03 -0400 +Message-Id: <1412355483-7670-1-git-send-email-aclements@csail.mit.edu> +X-Mailer: git-send-email 2.1.0 +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFlrLIsWRmVeSWpSXmKPExsUixCmqrLvlql6IwYIlahZT1jxmtbh+cyaz + A5PHkdeb2TyerbrFHMAUxWWTkpqTWZZapG+XwJXRO1+3YK9JxcnWJawNjJs1uxg5OCQETCSm + H5TtYuQEMsUkLtxbzwZiCwnMZpK4cjSni5ELyN7AKNHzeB8bhHOdSWLm7EWMEM4SRonJ+zYw + g7SwCehLrFg7iRXEFhGQlth5dzaYzSzgLPHw80Y2kG3CQNv2vVcHCbMIqEpsutDJCBLmFXCT + uPsgD+IIOYkNu/8zTmDkXcDIsIpRNiW3Sjc3MTOnODVZtzg5MS8vtUjXSC83s0QvNaV0EyM4 + JCR5dzC+O6h0iFGAg1GJh/fDDd0QIdbEsuLK3EOMkhxMSqK8sVf0QoT4kvJTKjMSizPii0pz + UosPMUpwMCuJ8C5YCZTjTUmsrEotyodJSXOwKInzbvrBFyIkkJ5YkpqdmlqQWgSTleHgUJLg + LQUZKliUmp5akZaZU4KQZuLgBBnOAzQ8HaSGt7ggMbc4Mx0if4pRl2PSxve9TEIsefl5qVLi + vIsuAxUJgBRllObBzYHF8itGcaC3hHnzQUbxANMA3KRXQEuYgJa8s9cFWVKSiJCSamDMuJsr + bRp+TGrdofwLgdka97edEE3YeulqjXbsrmei78z/tb35rTxPRu8Y0yqhyc9LcpLNv7P9YSvR + yqhYsl5TzHeHcdheDkmRRzONf969L7b7qmVPf43uv63h2yd5fJzz3PK7rz1bCWPCxwsdPtf0 + E4t+cLrUVc15frXfac+FyBLubYfOz2VWYinOSDTUYi4qTgQAc5E+acACAAA= +Cc: Austin Clements +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Fri, 03 Oct 2014 16:58:36 -0000 + +Previously, this was implemented using a horrible GDB script (because +there is no such thing as a non-horrible GDB script). This GDB script +often broke with newer versions of GDB for mysterious reasons. Port +the test script to GDB's Python API, which makes the code much cleaner +and, hopefully, more stable. +--- + +Hi Amadeusz. Does this patch fix the problem for you? I don't have +GDB 7.8, so I can't test against it, but the Python interface is less +fragile than the GDB scripting language. (Even if this doesn't fix +your problem, I think we should switch to the Python interface.) + + test/T380-atomicity.sh | 2 +- + test/atomicity.gdb | 54 -------------------------------------- + test/atomicity.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 72 insertions(+), 55 deletions(-) + delete mode 100644 test/atomicity.gdb + create mode 100644 test/atomicity.py + +diff --git a/test/T380-atomicity.sh b/test/T380-atomicity.sh +index 2daef90..ee1e2f4 100755 +--- a/test/T380-atomicity.sh ++++ b/test/T380-atomicity.sh +@@ -64,7 +64,7 @@ if test_require_external_prereq gdb; then + # -tty /dev/null works around a conflict between the 'timeout' wrapper + # and gdb's attempt to control the TTY. + export MAIL_DIR +- gdb -tty /dev/null -batch -x $TEST_DIRECTORY/atomicity.gdb notmuch 1>gdb.out 2>&1 ++ gdb -tty /dev/null -batch -x $TEST_DIRECTORY/atomicity.py notmuch 1>gdb.out 2>&1 + + # Get the final, golden output + notmuch search '*' > expected +diff --git a/test/atomicity.gdb b/test/atomicity.gdb +deleted file mode 100644 +index 15adb16..0000000 +--- a/test/atomicity.gdb ++++ /dev/null +@@ -1,54 +0,0 @@ +-# This gdb script runs notmuch new and simulates killing and +-# restarting notmuch new after every Xapian commit. To simulate this +-# more efficiently, this script runs notmuch new and, immediately +-# after every Xapian commit, it *pauses* the running notmuch new, +-# copies the entire database and maildir to a snapshot directory, and +-# executes a full notmuch new on that snapshot, comparing the final +-# results with the expected output. It can then resume the paused +-# notmuch new, which is still running on the original maildir, and +-# repeat this process. +- +-set args new +- +-# Make Xapian commit after every operation instead of batching +-set environment XAPIAN_FLUSH_THRESHOLD = 1 +- +-# gdb can't keep track of a simple integer. This is me weeping. +-shell echo 0 > outcount +- +-shell touch inodes +- +-# work around apparent issue with lazy library loading on some +-# platforms +-set breakpoint pending on +- +-break rename +-commands +-# As an optimization, only consider snapshots after a Xapian commit. +-# Xapian overwrites record.base? as the last step in the commit. +-shell echo > gdbcmd +-shell stat -c %i $MAIL_DIR/.notmuch/xapian/record.base* > inodes.new +-shell if cmp inodes inodes.new; then echo cont > gdbcmd; fi +-shell mv inodes.new inodes +-source gdbcmd +- +-# Save a backtrace in case the test does fail +-set logging file backtrace +-set logging on +-backtrace +-set logging off +-shell mv backtrace backtrace.`cat outcount` +- +-# Snapshot the database +-shell rm -r $MAIL_DIR.snap/.notmuch +-shell cp -r $MAIL_DIR/.notmuch $MAIL_DIR.snap/.notmuch +-# Restore the mtime of $MAIL_DIR.snap, which we just changed +-shell touch -r $MAIL_DIR $MAIL_DIR.snap +-# Run notmuch new to completion on the snapshot +-shell NOTMUCH_CONFIG=${NOTMUCH_CONFIG}.snap XAPIAN_FLUSH_THRESHOLD=1000 notmuch new > /dev/null +-shell NOTMUCH_CONFIG=${NOTMUCH_CONFIG}.snap notmuch search '*' > search.`cat outcount` 2>&1 +-shell echo $(expr $(cat outcount) + 1) > outcount +-cont +-end +- +-run +diff --git a/test/atomicity.py b/test/atomicity.py +new file mode 100644 +index 0000000..01a4205 +--- /dev/null ++++ b/test/atomicity.py +@@ -0,0 +1,71 @@ ++# This gdb Python script runs notmuch new and simulates killing and ++# restarting notmuch new after every Xapian commit. To simulate this ++# more efficiently, this script runs notmuch new and, immediately ++# after every Xapian commit, it *pauses* the running notmuch new, ++# copies the entire database and maildir to a snapshot directory, and ++# executes a full notmuch new on that snapshot, comparing the final ++# results with the expected output. It can then resume the paused ++# notmuch new, which is still running on the original maildir, and ++# repeat this process. ++ ++import gdb ++import os ++import glob ++import shutil ++import subprocess ++ ++gdb.execute('set args new') ++ ++# Make Xapian commit after every operation instead of batching ++gdb.execute('set environment XAPIAN_FLUSH_THRESHOLD = 1') ++ ++maildir = os.environ['MAIL_DIR'] ++ ++# Trap calls to rename, which happens just before Xapian commits ++class RenameBreakpoint(gdb.Breakpoint): ++ def __init__(self, *args, **kwargs): ++ super(RenameBreakpoint, self).__init__(*args, **kwargs) ++ self.last_inodes = {} ++ self.n = 0 ++ ++ def stop(self): ++ # As an optimization, only consider snapshots after a Xapian ++ # has really committed. Xapian overwrites record.base? as the ++ # last step in the commit, so keep an eye on their inumbers. ++ inodes = {} ++ for path in glob.glob('%s/.notmuch/xapian/record.base*' % maildir): ++ inodes[path] = os.stat(path).st_ino ++ if inodes == self.last_inodes: ++ # Continue ++ return False ++ self.last_inodes = inodes ++ ++ # Save a backtrace in case the test does fail ++ backtrace = gdb.execute('backtrace', to_string=True) ++ open('backtrace.%d' % self.n, 'w').write(backtrace) ++ ++ # Snapshot the database ++ shutil.rmtree('%s.snap/.notmuch' % maildir) ++ shutil.copytree('%s/.notmuch' % maildir, '%s.snap/.notmuch' % maildir) ++ # Restore the mtime of $MAIL_DIR.snap/ ++ shutil.copystat('%s/.notmuch' % maildir, '%s.snap/.notmuch' % maildir) ++ ++ # Run notmuch new to completion on the snapshot ++ env = os.environ.copy() ++ env.update(NOTMUCH_CONFIG=os.environ['NOTMUCH_CONFIG'] + '.snap', ++ XAPIAN_FLUSH_THRESHOLD='1000') ++ subprocess.check_call( ++ ['notmuch', 'new'], env=env, stdout=open('/dev/null', 'w')) ++ subprocess.check_call( ++ ['notmuch', 'search', '*'], env=env, ++ stdout=open('search.%d' % self.n, 'w')) ++ ++ # Tell the shell how far we've gotten ++ open('outcount', 'w').write(str(self.n + 1)) ++ ++ # Continue ++ self.n += 1 ++ return False ++RenameBreakpoint('rename') ++ ++gdb.execute('run') +-- +2.1.0 + -- 2.26.2