--- /dev/null
+Return-Path: <ethan.glasser.camp@gmail.com>\r
+X-Original-To: notmuch@notmuchmail.org\r
+Delivered-To: notmuch@notmuchmail.org\r
+Received: from localhost (localhost [127.0.0.1])\r
+ by olra.theworths.org (Postfix) with ESMTP id 4A161431FBD\r
+ for <notmuch@notmuchmail.org>; Fri, 4 Oct 2013 18:28:21 -0700 (PDT)\r
+X-Virus-Scanned: Debian amavisd-new at olra.theworths.org\r
+X-Spam-Flag: NO\r
+X-Spam-Score: -0.799\r
+X-Spam-Level: \r
+X-Spam-Status: No, score=-0.799 tagged_above=-999 required=5\r
+ tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, DKIM_VALID_AU=-0.1,\r
+ FREEMAIL_FROM=0.001, RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled\r
+Received: from olra.theworths.org ([127.0.0.1])\r
+ by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024)\r
+ with ESMTP id rdQfzwQdBW4y for <notmuch@notmuchmail.org>;\r
+ Fri, 4 Oct 2013 18:28:16 -0700 (PDT)\r
+Received: from mail-qa0-f44.google.com (mail-qa0-f44.google.com\r
+ [209.85.216.44]) (using TLSv1 with cipher RC4-SHA (128/128 bits))\r
+ (No client certificate requested)\r
+ by olra.theworths.org (Postfix) with ESMTPS id 52FBE431FAF\r
+ for <notmuch@notmuchmail.org>; Fri, 4 Oct 2013 18:28:16 -0700 (PDT)\r
+Received: by mail-qa0-f44.google.com with SMTP id j7so1607348qaq.10\r
+ for <notmuch@notmuchmail.org>; Fri, 04 Oct 2013 18:28:13 -0700 (PDT)\r
+DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113;\r
+ h=from:to:subject:in-reply-to:references:user-agent:date:message-id\r
+ :mime-version:content-type;\r
+ bh=yHoDnFNlw3o30xsgKwGZhEWp0L6xhXNl363yAv1oSYQ=;\r
+ b=OtDmvQM83JHSad6l3VssCmOuHRQMma4zgX03PQVIRmOFkGx2P5rCMcAeY3GVV2GfR3\r
+ hGFIn1bi7YksArkz/KNiU7S0QunkgTriKcsepHDsnzgenZjYRoU2EFV7O4osxnLVMKRB\r
+ UQ7Ymc79Re8AvsID33U2439zey2I7iNV8o3pKwlOFGoPQlRqvQzkFAE+dOQzRwpMg26q\r
+ 4vqlfjsfjTCPie5mIp+rgpeB0MoQkYXw0Qy5g2Ofxl5RK0+pxcAKISCSqyJxR9m0h9KP\r
+ 3hSmfq5lCvgVgpi4OBHhjUvca82TjLibGXWc9r0cNfb/wcWU1+17L0+QwPlqzSm6PfBB\r
+ zeVw==\r
+X-Received: by 10.224.130.72 with SMTP id r8mr22741562qas.32.1380936493626;\r
+ Fri, 04 Oct 2013 18:28:13 -0700 (PDT)\r
+Received: from smtp.gmail.com ([66.114.71.21])\r
+ by mx.google.com with ESMTPSA id x8sm35331978qam.2.1969.12.31.16.00.00\r
+ (version=TLSv1.2 cipher=RC4-SHA bits=128/128);\r
+ Fri, 04 Oct 2013 18:28:12 -0700 (PDT)\r
+From: Ethan Glasser-Camp <ethan.glasser.camp@gmail.com>\r
+To: David Bremner <david@tethera.net>,\r
+ notmuch mailing list <notmuch@notmuchmail.org>\r
+Subject: Re: On disk tag storage format\r
+In-Reply-To: <87vc9mtpxh.fsf@zancas.localnet>\r
+References: <874nk8v9zw.fsf@zancas.localnet> <87vc9mtpxh.fsf@zancas.localnet>\r
+User-Agent: Notmuch/0.16+80~g81ee785 (http://notmuchmail.org) Emacs/24.2.1\r
+ (x86_64-pc-linux-gnu)\r
+Date: Fri, 04 Oct 2013 21:28:10 -0400\r
+Message-ID: <87fvsgh5g5.fsf@betacantrips.com>\r
+MIME-Version: 1.0\r
+Content-Type: multipart/mixed; boundary="=-=-="\r
+X-BeenThere: notmuch@notmuchmail.org\r
+X-Mailman-Version: 2.1.13\r
+Precedence: list\r
+List-Id: "Use and development of the notmuch mail system."\r
+ <notmuch.notmuchmail.org>\r
+List-Unsubscribe: <http://notmuchmail.org/mailman/options/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=unsubscribe>\r
+List-Archive: <http://notmuchmail.org/pipermail/notmuch>\r
+List-Post: <mailto:notmuch@notmuchmail.org>\r
+List-Help: <mailto:notmuch-request@notmuchmail.org?subject=help>\r
+List-Subscribe: <http://notmuchmail.org/mailman/listinfo/notmuch>,\r
+ <mailto:notmuch-request@notmuchmail.org?subject=subscribe>\r
+X-List-Received-Date: Sat, 05 Oct 2013 01:28:21 -0000\r
+\r
+--=-=-=\r
+Content-Type: text/plain\r
+\r
+David Bremner <david@tethera.net> writes:\r
+\r
+> It's still a prototype, and there is not much error checking, and there\r
+> are certain issues not dealt with at all (the ones I thought about are\r
+> commented).\r
+\r
+Hi everyone,\r
+\r
+I'm very interested in running notmuch on all my laptops and having my\r
+mail and its tags be synchronized for me, so at Bremner's direction on\r
+IRC, I played around with this script a little. At first it wouldn't run\r
+on my computer; the script uses message IDs as filenames, which can be\r
+quite long, whereas I keep my mail in my $HOME, which is on an ecryptfs\r
+filesystem, and has a filename limit of 143 characters.\r
+\r
+I've modified the script so that it would run by mangling filenames,\r
+which is irreversible (the original tried to encode/decode filenames\r
+reversibly). Then I got a little carried away, adding --verbose and\r
+--dry-run options as well as removing a couple trailing\r
+semicolons. Here's my version, in case it should interest anyone else.\r
+\r
+\r
+--=-=-=\r
+Content-Type: text/x-python\r
+Content-Disposition: inline; filename=linksync.py\r
+Content-Description: linksync.py\r
+\r
+# Copyright 2013, David Bremner <david@tethera.net>\r
+\r
+# Licensed under the same terms as notmuch.\r
+\r
+import notmuch\r
+import re\r
+import os, errno\r
+import sys\r
+from collections import defaultdict\r
+import argparse\r
+import hashlib\r
+\r
+# skip automatic and maildir tags\r
+\r
+skiptags = re.compile(r"^(attachement|signed|encrypted|draft|flagged|passed|replied|unread)$")\r
+\r
+# some random person on stack overflow suggests:\r
+\r
+def mkdir_p(path):\r
+ try:\r
+ os.makedirs(path)\r
+ except OSError as exc: # Python >2.5\r
+ if exc.errno == errno.EEXIST and os.path.isdir(path):\r
+ pass\r
+ else: raise\r
+\r
+VERBOSE = False\r
+\r
+def log(msg):\r
+ if VERBOSE:\r
+ print(msg)\r
+\r
+CHARSET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+_@=.,-'\r
+\r
+encode_re = '([^{0}])'.format(CHARSET)\r
+\r
+decode_re = '[%]([0-7][0-9A-Fa-f])'\r
+\r
+def encode_one_char(match):\r
+ return('%{:02x}'.format(ord(match.group(1))))\r
+\r
+def encode_for_fs(str):\r
+ return re.sub(encode_re,encode_one_char, str,0)\r
+\r
+def mangle_message_id(msg_id):\r
+ """\r
+ Return a mangled version of the message id, suitable for use as a filename.\r
+ """\r
+ MAX_LENGTH = 143\r
+ FLAGS_LENGTH = 8 # :2,S...??\r
+ encoded = encode_for_fs(msg_id)\r
+ if len(encoded) < MAX_LENGTH - FLAGS_LENGTH:\r
+ return encoded\r
+\r
+ SHA_LENGTH = 8\r
+ TRUNCATED_ID_LENGTH = MAX_LENGTH - SHA_LENGTH - FLAGS_LENGTH\r
+ PREFIX_LENGTH = SUFFIX_LENGTH = (TRUNCATED_ID_LENGTH - 3) // 2\r
+ prefix = encoded[:PREFIX_LENGTH]\r
+ suffix = encoded[-SUFFIX_LENGTH:]\r
+ sha = hashlib.sha256()\r
+ sha.update(encoded)\r
+ return prefix + '...' + suffix + sha.hexdigest()[:SHA_LENGTH]\r
+\r
+def decode_one_char(match):\r
+ return chr(int(match.group(1),16))\r
+\r
+def decode_from_fs(str):\r
+ return re.sub(decode_re,decode_one_char, str, 0)\r
+\r
+def mk_tag_dir(tagdir):\r
+\r
+ mkdir_p (os.path.join(tagdir, 'cur'))\r
+ mkdir_p (os.path.join(tagdir, 'new'))\r
+ mkdir_p (os.path.join(tagdir, 'tmp'))\r
+\r
+\r
+flagpart = '(:2,[^:]*)'\r
+flagre = re.compile(flagpart + '$')\r
+\r
+def path_for_msg (dir, msg):\r
+ filename = msg.get_filename()\r
+ flagsmatch = flagre.search(filename)\r
+ if flagsmatch == None:\r
+ flags = ''\r
+ else:\r
+ flags = flagsmatch.group(1)\r
+\r
+ return os.path.join(dir, 'cur', mangle_message_id(msg.get_message_id()) + flags)\r
+\r
+\r
+def unlink_message(dir, msg):\r
+\r
+ dir = os.path.join(dir, 'cur')\r
+\r
+ filepattern = mangle_filename_for_fs(msg.get_message_id()) + flagpart +'?$'\r
+\r
+ filere = re.compile(filepattern)\r
+\r
+ for file in os.listdir(dir):\r
+ if filere.match(file):\r
+ log("Unlinking {}".format(os.path.join(dir, file)))\r
+ if not opts.dry_run:\r
+ os.unlink(os.path.join(dir, file))\r
+\r
+def dir_for_tag(tag):\r
+ enc_tag = encode_for_fs (tag)\r
+ return os.path.join(tagroot, enc_tag)\r
+\r
+disk_tags = defaultdict(set)\r
+disk_ids = set()\r
+\r
+def read_tags_from_disk(rootdir):\r
+\r
+ for root, subFolders, files in os.walk(rootdir):\r
+ for filename in files:\r
+ mangled_id = filename.split(':')[0]\r
+ tag = root.split('/')[-2]\r
+ disk_ids.add(mangled_id)\r
+ disk_tags[mangled_id].add(decode_from_fs(tag))\r
+\r
+# Main program\r
+\r
+parser = argparse.ArgumentParser(description='Sync notmuch tag database to/from link farm')\r
+parser.add_argument('-l','--link-style',choices=['hard','symbolic', 'adaptive'],\r
+ default='adaptive')\r
+parser.add_argument('-d','--destination',choices=['disk','notmuch'], default='disk')\r
+parser.add_argument('-t','--threshold', default=50000L, type=int)\r
+parser.add_argument('-n','--dry-run', default=False, action='store_true')\r
+parser.add_argument('-v','--verbose', default=False, action='store_true')\r
+\r
+parser.add_argument('tagroot')\r
+\r
+opts=parser.parse_args()\r
+VERBOSE = opts.verbose\r
+\r
+tagroot=opts.tagroot\r
+\r
+sync_from_links = (opts.destination == 'notmuch')\r
+\r
+read_tags_from_disk(tagroot)\r
+\r
+if sync_from_links:\r
+ db = notmuch.Database(mode=notmuch.Database.MODE.READ_WRITE)\r
+else:\r
+ db = notmuch.Database(mode=notmuch.Database.MODE.READ_ONLY)\r
+\r
+dbtags = filter (lambda tag: not skiptags.match(tag), db.get_all_tags())\r
+\r
+querystr = ' OR '.join(map (lambda tag: 'tag:'+tag, dbtags))\r
+\r
+q_new = notmuch.Query(db, querystr)\r
+q_new.set_sort(notmuch.Query.SORT.UNSORTED)\r
+for msg in q_new.search_messages():\r
+\r
+ # silently ignore empty tags\r
+ db_tags = set(filter (lambda tag: tag != '' and not skiptags.match(tag),\r
+ msg.get_tags()))\r
+\r
+ message_id = msg.get_message_id()\r
+\r
+ mangled_id = mangle_message_id(message_id)\r
+\r
+ disk_ids.discard(mangled_id)\r
+\r
+ missing_on_disk = db_tags.difference(disk_tags[mangled_id])\r
+ missing_in_db = disk_tags[mangled_id].difference(db_tags)\r
+\r
+ if sync_from_links:\r
+ msg.freeze()\r
+\r
+ filename = msg.get_filename()\r
+\r
+ if len(missing_on_disk) > 0:\r
+ if opts.link_style == 'adaptive':\r
+ statinfo = os.stat (filename)\r
+ symlink = (statinfo.st_size > opts.threshold)\r
+ else:\r
+ symlink = opts.link_style == 'symbolic'\r
+\r
+ for tag in missing_on_disk:\r
+\r
+ if sync_from_links:\r
+ log("Removing tag {} from {}".format(tag, message_id))\r
+ if not opts.dry_run:\r
+ msg.remove_tag(tag,sync_maildir_flags=False)\r
+ else:\r
+ tagdir = dir_for_tag (tag)\r
+\r
+ if not opts.dry_run:\r
+ mk_tag_dir (tagdir)\r
+\r
+ newlink = path_for_msg (tagdir, msg)\r
+\r
+ log("Linking {} to {}".format(filename, newlink))\r
+ if not opts.dry_run:\r
+ if symlink:\r
+ os.symlink(filename, newlink)\r
+ else:\r
+ os.link(filename, newlink)\r
+\r
+\r
+ for tag in missing_in_db:\r
+ if sync_from_links:\r
+ log("Adding {} to message {}".format(tag, message_id))\r
+ if not opts.dry_run:\r
+ msg.add_tag(tag,sync_maildir_flags=False)\r
+ else:\r
+ tagdir = dir_for_tag (tag)\r
+ unlink_message(tagdir,msg)\r
+\r
+ if sync_from_links:\r
+ msg.thaw()\r
+\r
+# everything remaining in disk_ids is a deleted message\r
+# unless we are syncing back to the database, in which case\r
+# it just might not currently have any non maildir tags.\r
+\r
+if not sync_from_links:\r
+ for root, subFolders, files in os.walk(tagroot):\r
+ for filename in files:\r
+ mangled_id = filename.split(':')[0]\r
+ if mangled_id in disk_ids:\r
+ os.unlink(os.path.join(root, filename))\r
+\r
+\r
+db.close()\r
+\r
+# currently empty directories are not pruned.\r
+\r
+--=-=-=--\r
projects / notmuch-archives.git / commitdiff