From 63cdbd9beaa770590c8377bc16560cb3960bc247 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sun, 28 Apr 2013 00:24:28 +0300 Subject: [PATCH] Re: [PATCH v5 03/12] cli: add insert command --- b3/417724a1c1860b798d1c2e947e45aa820f3ab3 | 516 ++++++++++++++++++++++ 1 file changed, 516 insertions(+) create mode 100644 b3/417724a1c1860b798d1c2e947e45aa820f3ab3 diff --git a/b3/417724a1c1860b798d1c2e947e45aa820f3ab3 b/b3/417724a1c1860b798d1c2e947e45aa820f3ab3 new file mode 100644 index 000000000..b85a565b9 --- /dev/null +++ b/b3/417724a1c1860b798d1c2e947e45aa820f3ab3 @@ -0,0 +1,516 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id C7A10431FBF + for ; Sat, 27 Apr 2013 14:24:40 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id vCzTYHVxf+kc for ; + Sat, 27 Apr 2013 14:24:38 -0700 (PDT) +Received: from mail-la0-f49.google.com (mail-la0-f49.google.com + [209.85.215.49]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 6BE77431FAE + for ; Sat, 27 Apr 2013 14:24:38 -0700 (PDT) +Received: by mail-la0-f49.google.com with SMTP id fp13so2195942lab.8 + for ; Sat, 27 Apr 2013 14:24:36 -0700 (PDT) +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=google.com; s=20120113; + h=x-received:from:to:subject:in-reply-to:references:user-agent:date + :message-id:mime-version:content-type:content-transfer-encoding + :x-gm-message-state; + bh=I5GpV/6rgqYgKg0jeDb/C/UMiD5W7PYr4wO94Y3Ywz4=; + b=PfJvWoVOZE7iOcIco3x/sEXK39JPlYK0PQBj5pHiCbvZ364IhrwBmnKL7DVHvw6EcK + DFZbhmbZNsx9ZmJysicxqPTJykqlJ3fKEj3Zp6Y7+EdryCol13XZ77rqKNGE+LaTzD6y + 0r/s1iemlrZNc8k/X+eUtIPK9a7h9dF+n5C4BFw93gYbTZtESEQ/CNvldxNw+pD2/moU + RHm2ZeZBqzQf5YU78mwsZW5mQQ5ijRySwdjZnUnWfugouw1ssVDwu1JdcvmoKTj+mwft + Kj+J9UZn/7qSDSyhnDu4k6abhggRciQDVAtA1mWvzLGK7cU6Y0jlqvXVpBLEQ4vbllFp + DZ/g== +X-Received: by 10.112.129.137 with SMTP id nw9mr24536735lbb.56.1367097875460; + Sat, 27 Apr 2013 14:24:35 -0700 (PDT) +Received: from localhost (dsl-hkibrasgw2-58c376-211.dhcp.inet.fi. + [88.195.118.211]) + by mx.google.com with ESMTPSA id r9sm3509338lbr.3.2013.04.27.14.24.33 + for + (version=TLSv1.2 cipher=RC4-SHA bits=128/128); + Sat, 27 Apr 2013 14:24:34 -0700 (PDT) +From: Jani Nikula +To: Peter Wang , notmuch@notmuchmail.org +Subject: Re: [PATCH v5 03/12] cli: add insert command +In-Reply-To: <1364942517-6982-4-git-send-email-novalazy@gmail.com> +References: <1364942517-6982-1-git-send-email-novalazy@gmail.com> + <1364942517-6982-4-git-send-email-novalazy@gmail.com> +User-Agent: Notmuch/0.15.2+70~g2eeb96a (http://notmuchmail.org) Emacs/24.2.1 + (x86_64-pc-linux-gnu) +Date: Sun, 28 Apr 2013 00:24:28 +0300 +Message-ID: <8761z7r7jn.fsf@nikula.org> +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: quoted-printable +X-Gm-Message-State: + ALoCoQkTLEevSBETDEHJudgQoaJ4yi3L/7y2+FG6ldcU288Uridvy/LHIzVhrwO02MVgUqJtzZY/ +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sat, 27 Apr 2013 21:24:40 -0000 + +On Wed, 03 Apr 2013, Peter Wang wrote: +> The notmuch insert command reads a message from standard input, +> writes it to a Maildir folder, and then incorporates the message into +> the notmuch database. Essentially it moves the functionality of +> notmuch-deliver into notmuch. +> +> Though it could be used as an alternative to notmuch new, the reason +> I want this is to allow my notmuch frontend to add postponed or sent +> messages to the mail store and notmuch database, without resorting to +> another tool (e.g. notmuch-deliver) nor directly modifying the maildir. +> --- +> Makefile.local | 1 + +> notmuch-client.h | 3 + +> notmuch-insert.c | 336 +++++++++++++++++++++++++++++++++++++++++++++++++= +++++++ +> notmuch.c | 3 + +> 4 files changed, 343 insertions(+) +> create mode 100644 notmuch-insert.c +> +> diff --git a/Makefile.local b/Makefile.local +> index c274f07..bb2381d 100644 +> --- a/Makefile.local +> +++ b/Makefile.local +> @@ -261,6 +261,7 @@ notmuch_client_srcs =3D \ +> notmuch-config.c \ +> notmuch-count.c \ +> notmuch-dump.c \ +> + notmuch-insert.c \ +> notmuch-new.c \ +> notmuch-reply.c \ +> notmuch-restore.c \ +> diff --git a/notmuch-client.h b/notmuch-client.h +> index 45749a6..da332f3 100644 +> --- a/notmuch-client.h +> +++ b/notmuch-client.h +> @@ -180,6 +180,9 @@ int +> notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]); +>=20=20 +> int +> +notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]= +); +> + +> +int +> notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[]); +>=20=20 +> int +> diff --git a/notmuch-insert.c b/notmuch-insert.c +> new file mode 100644 +> index 0000000..19b1cf9 +> --- /dev/null +> +++ b/notmuch-insert.c +> @@ -0,0 +1,336 @@ +> +/* notmuch - Not much of an email program, (just index and search) +> + * +> + * Copyright =C2=A9 2013 Peter Wang +> + * +> + * Based in part on notmuch-deliver +> + * Copyright =C2=A9 2010 Ali Polatel +> + * +> + * This program is free software: you can redistribute it and/or modify +> + * it under the terms of the GNU General Public License as published by +> + * the Free Software Foundation, either version 3 of the License, or +> + * (at your option) any later version. +> + * +> + * This program is distributed in the hope that it will be useful, +> + * but WITHOUT ANY WARRANTY; without even the implied warranty of +> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +> + * GNU General Public License for more details. +> + * +> + * You should have received a copy of the GNU General Public License +> + * along with this program. If not, see http://www.gnu.org/licenses/ . +> + * +> + * Author: Peter Wang +> + */ +> + +> +#include "notmuch-client.h" +> +#include "tag-util.h" +> + +> +#include +> +#include +> +#include +> + +> +static volatile sig_atomic_t interrupted; +> + +> +static void +> +handle_sigint (unused (int sig)) +> +{ +> + static char msg[] =3D "Stopping... \n"; +> + +> + /* This write is "opportunistic", so it's okay to ignore the +> + * result. It is not required for correctness, and if it does +> + * fail or produce a short write, we want to get out of the signal +> + * handler as quickly as possible, not retry it. */ +> + IGNORE_RESULT (write (2, msg, sizeof (msg) - 1)); +> + interrupted =3D 1; +> +} +> + +> +/* Like gethostname but guarantees that a null-terminated hostname is +> + * returned, even if it has to make one up. Invalid characters are +> + * substituted such that the hostname can be used within a filename. +> + */ +> +static void +> +safe_gethostname (char *hostname, size_t len) +> +{ +> + char *p; +> + +> + if (gethostname (hostname, len) =3D=3D -1) { +> + strncpy (hostname, "unknown", len); +> + } +> + hostname[len - 1] =3D '\0'; +> + +> + for (p =3D hostname; *p !=3D '\0'; p++) { +> + if (*p =3D=3D '/' || *p =3D=3D ':') +> + *p =3D '_'; +> + } +> +} +> + +> +/* Call fsync() on a directory path. */ +> +static notmuch_bool_t +> +sync_dir (const char *dir) +> +{ +> + notmuch_bool_t ret; +> + int fd; +> + +> + fd =3D open (dir, O_RDONLY); +> + if (fd =3D=3D -1) { +> + fprintf (stderr, "Error: open() dir failed: %s\n", strerror (errno)); +> + return FALSE; +> + } +> + ret =3D (fsync (fd) =3D=3D 0); +> + if (! ret) { +> + fprintf (stderr, "Error: fsync() dir failed: %s\n", strerror (errno)); +> + } +> + close (fd); +> + return ret; +> +} +> + +> +/* Open a unique file in the Maildir 'tmp' directory. +> + * Returns the file descriptor on success, or -1 on failure. +> + * On success, file paths for the message in the 'tmp' and 'new' +> + * directories are returned via tmppath and newpath, +> + * and the path of the 'new' directory itself in newdir. */ +> +static int +> +maildir_open_tmp_file (void *ctx, const char *dir, +> + char **tmppath, char **newpath, char **newdir) +> +{ +> + pid_t pid; +> + char hostname[256]; +> + struct timeval tv; +> + char *filename; +> + int fd =3D -1; +> + +> + /* We follow the Dovecot file name generation algorithm. */ +> + pid =3D getpid (); +> + safe_gethostname (hostname, sizeof (hostname)); +> + do { +> + gettimeofday (&tv, NULL); +> + filename =3D talloc_asprintf (ctx, "%ld.M%ldP%d.%s", +> + tv.tv_sec, tv.tv_usec, pid, hostname); +> + if (! filename) { +> + fprintf (stderr, "Out of memory\n"); +> + return -1; +> + } +> + +> + *tmppath =3D talloc_asprintf (ctx, "%s/tmp/%s", dir, filename); +> + if (! *tmppath) { +> + fprintf (stderr, "Out of memory\n"); +> + return -1; +> + } +> + +> + fd =3D open (*tmppath, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600); +> + } while (fd =3D=3D -1 && errno =3D=3D EEXIST); +> + +> + if (fd =3D=3D -1) { +> + fprintf (stderr, "Error: opening %s: %s\n", *tmppath, strerror (errno)); +> + return -1; +> + } +> + +> + *newdir =3D talloc_asprintf (ctx, "%s/new", dir); +> + *newpath =3D talloc_asprintf (ctx, "%s/new/%s", dir, filename); +> + if (! *newdir || ! *newpath) { +> + fprintf (stderr, "Out of memory\n"); +> + close (fd); +> + unlink (*tmppath); +> + return -1; +> + } +> + +> + talloc_free (filename); +> + +> + return fd; +> +} +> + +> +/* Copy the contents of standard input (fdin) into fdout. */ +> +static notmuch_bool_t +> +copy_stdin (int fdin, int fdout) +> +{ +> + char buf[4096]; +> + char *p; +> + ssize_t remain; +> + ssize_t written; +> + +> + while (! interrupted) { +> + remain =3D read (fdin, buf, sizeof (buf)); +> + if (remain =3D=3D 0) +> + break; +> + if (remain < 0) { +> + if (errno =3D=3D EINTR) +> + continue; +> + fprintf (stderr, "Error: reading from standard input: %s\n", +> + strerror (errno)); +> + return FALSE; +> + } +> + +> + p =3D buf; +> + do { +> + written =3D write (fdout, p, remain); +> + if (written < 0 && errno =3D=3D EINTR) +> + continue; +> + if (written <=3D 0) { +> + fprintf (stderr, "Error: writing to temporary file: %s", +> + strerror (errno)); +> + return FALSE; +> + } +> + p +=3D written; +> + remain -=3D written; +> + } while (remain > 0); +> + } +> + +> + return ! interrupted; +> +} +> + +> +/* Add the specified message file to the notmuch database, applying tags. +> + * The file is renamed to encode notmuch tags as maildir flags. */ +> +static notmuch_bool_t +> +add_file_to_database (notmuch_database_t *notmuch, const char *path, +> + tag_op_list_t *tag_ops) +> +{ +> + notmuch_message_t *message; +> + notmuch_status_t status; +> + +> + status =3D notmuch_database_add_message (notmuch, path, &message); +> + switch (status) { +> + case NOTMUCH_STATUS_SUCCESS: +> + case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: +> + break; +> + default: +> + case NOTMUCH_STATUS_FILE_NOT_EMAIL: + +If such a message really arrives, the mail system will keep trying if +failure is returned. Maybe deliver the file without indexing, and return +success? + +> + case NOTMUCH_STATUS_READ_ONLY_DATABASE: +> + case NOTMUCH_STATUS_XAPIAN_EXCEPTION: +> + case NOTMUCH_STATUS_OUT_OF_MEMORY: +> + case NOTMUCH_STATUS_FILE_ERROR: +> + case NOTMUCH_STATUS_NULL_POINTER: +> + case NOTMUCH_STATUS_TAG_TOO_LONG: +> + case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: +> + case NOTMUCH_STATUS_UNBALANCED_ATOMIC: +> + case NOTMUCH_STATUS_LAST_STATUS: +> + fprintf (stderr, "Error: failed to add `%s' to notmuch database: %s\n", +> + path, notmuch_status_to_string (status)); +> + return FALSE; +> + } +> + +> + if (status =3D=3D NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) { +> + /* Don't change tags of an existing message. */ +> + status =3D notmuch_message_tags_to_maildir_flags (message); +> + if (status !=3D NOTMUCH_STATUS_SUCCESS) +> + fprintf (stderr, "Error: failed to sync tags to maildir flags\n"); +> + } else { +> + status =3D tag_op_list_apply (message, tag_ops, TAG_FLAG_MAILDIR_SYNC); + +Syncing tags to maildir flags is more interesting here than above. And +it should be done because notmuch insert allows arbitrary tags on the +command line. Having, for example, -unread or +flagged on the command +line makes the flags go out of sync. (notmuch new should do the syncing +too, but it's less important because it only adds new.tags.) + +However, calling notmuch_message_tags_to_maildir_flags() may rename the +file from new to cur, which blows up the directory syncing and file +unlinking on the error path in insert_message() below. + +> + } +> + +> + notmuch_message_destroy (message); +> + +> + return (status =3D=3D NOTMUCH_STATUS_SUCCESS); +> +} +> + +> +static notmuch_bool_t +> +insert_message (void *ctx, notmuch_database_t *notmuch, int fdin, +> + const char *dir, tag_op_list_t *tag_ops) +> +{ +> + char *tmppath; +> + char *newpath; +> + char *newdir; +> + int fdout; +> + char *cleanup_path; +> + +> + fdout =3D maildir_open_tmp_file (ctx, dir, &tmppath, &newpath, &newd= +ir); +> + if (fdout < 0) +> + return FALSE; +> + +> + cleanup_path =3D tmppath; +> + +> + if (! copy_stdin (fdin, fdout)) +> + goto FAIL; +> + +> + if (fsync (fdout) !=3D 0) { +> + fprintf (stderr, "Error: fsync failed: %s\n", strerror (errno)); +> + goto FAIL; +> + } +> + +> + close (fdout); +> + fdout =3D -1; +> + +> + /* Atomically move the new message file from the Maildir 'tmp' direc= +tory +> + * to the 'new' directory. We follow the Dovecot recommendation to +> + * simply use rename() instead of link() and unlink(). +> + * See also: http://wiki.dovecot.org/MailboxFormat/Maildir#Mail_deli= +very +> + */ +> + if (rename (tmppath, newpath) !=3D 0) { +> + fprintf (stderr, "Error: rename() failed: %s\n", strerror (errno)); +> + goto FAIL; +> + } +> + +> + cleanup_path =3D newpath; +> + +> + if (! add_file_to_database (notmuch, newpath, tag_ops)) { +> + /* XXX add an option to keep the file in maildir? */ + +Possibly a good idea to let the user decide. This is the part that I +worry most about in the series. It's not unusual to hit xapian +exceptions when the database is already locked, which results +(hopefully!) in another attempt at delivery. If not, mail is lost. + +However, keeping the file in maildir on indexing errors ignores the tags +specified on the notmuch insert command line, and the message will get +just new.tags next time notmuch new is run. This is not nice either. + +BR, +Jani. + +> + goto FAIL; +> + } +> + +> + if (! sync_dir (newdir)) +> + goto FAIL; +> + +> + return TRUE; +> + +> + FAIL: +> + if (fdout >=3D 0) +> + close (fdout); +> + unlink (cleanup_path); +> + return FALSE; +> +} +> + +> +int +> +notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) +> +{ +> + notmuch_database_t *notmuch; +> + struct sigaction action; +> + const char *db_path; +> + const char **new_tags; +> + size_t new_tags_length; +> + tag_op_list_t *tag_ops; +> + char *query_string =3D NULL; +> + const char *maildir; +> + int opt_index =3D 1; +> + unsigned int i; +> + notmuch_bool_t ret; +> + +> + db_path =3D notmuch_config_get_database_path (config); +> + new_tags =3D notmuch_config_get_new_tags (config, &new_tags_length); +> + +> + tag_ops =3D tag_op_list_create (config); +> + if (tag_ops =3D=3D NULL) { +> + fprintf (stderr, "Out of memory.\n"); +> + return 1; +> + } +> + for (i =3D 0; i < new_tags_length; i++) { +> + if (tag_op_list_append (tag_ops, new_tags[i], FALSE)) +> + return 1; +> + } +> + +> + if (parse_tag_command_line (config, argc - opt_index, argv + opt_ind= +ex, +> + &query_string, tag_ops)) +> + return 1; +> + +> + if (*query_string !=3D '\0') { +> + fprintf (stderr, "Error: unexpected query string: %s\n", query_string); +> + return 1; +> + } +> + +> + maildir =3D db_path; +> + +> + /* Setup our handler for SIGINT. We do not set SA_RESTART so that co= +pying +> + * from standard input may be interrupted. */ +> + memset (&action, 0, sizeof (struct sigaction)); +> + action.sa_handler =3D handle_sigint; +> + sigemptyset (&action.sa_mask); +> + action.sa_flags =3D 0; +> + sigaction (SIGINT, &action, NULL); +> + +> + if (notmuch_database_open (notmuch_config_get_database_path (config), +> + NOTMUCH_DATABASE_MODE_READ_WRITE, ¬much)) +> + return 1; +> + +> + ret =3D insert_message (config, notmuch, STDIN_FILENO, maildir, tag_= +ops); +> + +> + notmuch_database_destroy (notmuch); +> + +> + return (ret) ? 0 : 1; +> +} +> diff --git a/notmuch.c b/notmuch.c +> index f51a84f..1a44f11 100644 +> --- a/notmuch.c +> +++ b/notmuch.c +> @@ -48,6 +48,9 @@ static command_t commands[] =3D { +> { "new", notmuch_new_command, FALSE, +> "[options...]", +> "Find and import new messages to the notmuch database." }, +> + { "insert", notmuch_insert_command, FALSE, +> + "[options...] [--] [+|- ...] < message", +> + "Add a new message into the maildir and notmuch database." }, +> { "search", notmuch_search_command, FALSE, +> "[options...] [...]", +> "Search for messages matching the given search terms." }, +> --=20 +> 1.7.12.1 +> +> _______________________________________________ +> notmuch mailing list +> notmuch@notmuchmail.org +> http://notmuchmail.org/mailman/listinfo/notmuch -- 2.26.2