From 5f310fddd63c4da3ab47726a4a931e68626ed3c1 Mon Sep 17 00:00:00 2001 From: Mark Walters Date: Sat, 22 Jun 2013 08:48:56 +0100 Subject: [PATCH] Re: [PATCH v6 03/12] cli: add insert command --- c4/26e49ffe19c6f1765898846090b60f26af253a | 504 ++++++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 c4/26e49ffe19c6f1765898846090b60f26af253a diff --git a/c4/26e49ffe19c6f1765898846090b60f26af253a b/c4/26e49ffe19c6f1765898846090b60f26af253a new file mode 100644 index 000000000..145818c44 --- /dev/null +++ b/c4/26e49ffe19c6f1765898846090b60f26af253a @@ -0,0 +1,504 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 43442431FB6 + for ; Sat, 22 Jun 2013 00:49:10 -0700 (PDT) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -1.098 +X-Spam-Level: +X-Spam-Status: No, score=-1.098 tagged_above=-999 required=5 + tests=[DKIM_ADSP_CUSTOM_MED=0.001, FREEMAIL_FROM=0.001, + NML_ADSP_CUSTOM_MED=1.2, RCVD_IN_DNSWL_MED=-2.3] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id zTzfilsm46Rd for ; + Sat, 22 Jun 2013 00:49:02 -0700 (PDT) +Received: from mail2.qmul.ac.uk (mail2.qmul.ac.uk [138.37.6.6]) + (using TLSv1 with cipher DHE-RSA-AES256-SHA (256/256 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 8907B431FAE + for ; Sat, 22 Jun 2013 00:49:01 -0700 (PDT) +Received: from smtp.qmul.ac.uk ([138.37.6.40]) + by mail2.qmul.ac.uk with esmtp (Exim 4.71) + (envelope-from ) + id 1UqIZ9-0007eH-MF; Sat, 22 Jun 2013 08:49:00 +0100 +Received: from 93-97-24-31.zone5.bethere.co.uk ([93.97.24.31] helo=localhost) + by smtp.qmul.ac.uk with esmtpsa (TLSv1:AES128-SHA:128) (Exim 4.71) + (envelope-from ) + id 1UqIZ8-0001Du-VH; Sat, 22 Jun 2013 08:48:59 +0100 +From: Mark Walters +To: Peter Wang , notmuch@notmuchmail.org +Subject: Re: [PATCH v6 03/12] cli: add insert command +In-Reply-To: <1371880008-18312-4-git-send-email-novalazy@gmail.com> +References: <1371880008-18312-1-git-send-email-novalazy@gmail.com> + <1371880008-18312-4-git-send-email-novalazy@gmail.com> +User-Agent: Notmuch/0.15.2+171~ge2f30a2 (http://notmuchmail.org) Emacs/23.4.1 + (x86_64-pc-linux-gnu) +Date: Sat, 22 Jun 2013 08:48:56 +0100 +Message-ID: <87hagqvb4n.fsf@qmul.ac.uk> +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: quoted-printable +X-Sender-Host-Address: 93.97.24.31 +X-QM-SPAM-Info: Sender has good ham record. :) +X-QM-Body-MD5: 1d3c093fd17cca711a7face32803b846 (of first 20000 bytes) +X-SpamAssassin-Score: -0.0 +X-SpamAssassin-SpamBar: / +X-SpamAssassin-Report: The QM spam filters have analysed this message to + determine if it is + spam. We require at least 5.0 points to mark a message as spam. + This message scored -0.0 points. + Summary of the scoring: + * 0.0 FREEMAIL_FROM Sender email is commonly abused enduser mail + provider * (markwalters1009[at]gmail.com) + * -0.0 AWL AWL: From: address is in the auto white-list +X-QM-Scan-Virus: ClamAV says the message is clean +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sat, 22 Jun 2013 07:49:10 -0000 + +On Sat, 22 Jun 2013, Peter Wang wrote: +> The notmuch insert command reads a message from standard input, +> writes it to a Maildir folder, and then incorporates the message into +> the notmuch database. Essentially it moves the functionality of +> notmuch-deliver into notmuch. +> +> Though it could be used as an alternative to notmuch new, the reason +> I want this is to allow my notmuch frontend to add postponed or sent +> messages to the mail store and notmuch database, without resorting to +> another tool (e.g. notmuch-deliver) nor directly modifying the maildir. +> --- +> Makefile.local | 1 + +> notmuch-client.h | 3 + +> notmuch-insert.c | 335 +++++++++++++++++++++++++++++++++++++++++++++++++= +++++++ +> notmuch.c | 3 + +> 4 files changed, 342 insertions(+) +> create mode 100644 notmuch-insert.c +> +> diff --git a/Makefile.local b/Makefile.local +> index 644623f..84043fe 100644 +> --- a/Makefile.local +> +++ b/Makefile.local +> @@ -261,6 +261,7 @@ notmuch_client_srcs =3D \ +> notmuch-config.c \ +> notmuch-count.c \ +> notmuch-dump.c \ +> + notmuch-insert.c \ +> notmuch-new.c \ +> notmuch-reply.c \ +> notmuch-restore.c \ +> diff --git a/notmuch-client.h b/notmuch-client.h +> index 4a3c7ac..dfe81e6 100644 +> --- a/notmuch-client.h +> +++ b/notmuch-client.h +> @@ -181,6 +181,9 @@ int +> notmuch_new_command (notmuch_config_t *config, int argc, char *argv[]); +>=20=20 +> int +> +notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]= +); +> + +> +int +> notmuch_reply_command (notmuch_config_t *config, int argc, char *argv[]); +>=20=20 +> int +> diff --git a/notmuch-insert.c b/notmuch-insert.c +> new file mode 100644 +> index 0000000..1495234 +> --- /dev/null +> +++ b/notmuch-insert.c +> @@ -0,0 +1,335 @@ +> +/* notmuch - Not much of an email program, (just index and search) +> + * +> + * Copyright =C2=A9 2013 Peter Wang +> + * +> + * Based in part on notmuch-deliver +> + * Copyright =C2=A9 2010 Ali Polatel +> + * +> + * This program is free software: you can redistribute it and/or modify +> + * it under the terms of the GNU General Public License as published by +> + * the Free Software Foundation, either version 3 of the License, or +> + * (at your option) any later version. +> + * +> + * This program is distributed in the hope that it will be useful, +> + * but WITHOUT ANY WARRANTY; without even the implied warranty of +> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +> + * GNU General Public License for more details. +> + * +> + * You should have received a copy of the GNU General Public License +> + * along with this program. If not, see http://www.gnu.org/licenses/ . +> + * +> + * Author: Peter Wang +> + */ +> + +> +#include "notmuch-client.h" +> +#include "tag-util.h" +> + +> +#include +> +#include +> +#include +> + +> +static volatile sig_atomic_t interrupted; +> + +> +static void +> +handle_sigint (unused (int sig)) +> +{ +> + static char msg[] =3D "Stopping... \n"; +> + +> + /* This write is "opportunistic", so it's okay to ignore the +> + * result. It is not required for correctness, and if it does +> + * fail or produce a short write, we want to get out of the signal +> + * handler as quickly as possible, not retry it. */ +> + IGNORE_RESULT (write (2, msg, sizeof (msg) - 1)); +> + interrupted =3D 1; +> +} +> + +> +/* Like gethostname but guarantees that a null-terminated hostname is +> + * returned, even if it has to make one up. Invalid characters are +> + * substituted such that the hostname can be used within a filename. +> + */ +> +static void +> +safe_gethostname (char *hostname, size_t len) +> +{ +> + char *p; +> + +> + if (gethostname (hostname, len) =3D=3D -1) { +> + strncpy (hostname, "unknown", len); +> + } +> + hostname[len - 1] =3D '\0'; +> + +> + for (p =3D hostname; *p !=3D '\0'; p++) { +> + if (*p =3D=3D '/' || *p =3D=3D ':') +> + *p =3D '_'; +> + } +> +} +> + +> +/* Call fsync() on a directory path. */ +> +static notmuch_bool_t +> +sync_dir (const char *dir) +> +{ +> + notmuch_bool_t ret; +> + int fd; +> + +> + fd =3D open (dir, O_RDONLY); +> + if (fd =3D=3D -1) { +> + fprintf (stderr, "Error: open() dir failed: %s\n", strerror (errno)); +> + return FALSE; +> + } +> + ret =3D (fsync (fd) =3D=3D 0); +> + if (! ret) { +> + fprintf (stderr, "Error: fsync() dir failed: %s\n", strerror (errno)); +> + } +> + close (fd); +> + return ret; +> +} +> + +> +/* Open a unique file in the Maildir 'tmp' directory. + +perhaps add "in the folder dir" or similar? + +> + * Returns the file descriptor on success, or -1 on failure. +> + * On success, file paths for the message in the 'tmp' and 'new' +> + * directories are returned via tmppath and newpath, +> + * and the path of the 'new' directory itself in newdir. */ +> +static int +> +maildir_open_tmp_file (void *ctx, const char *dir, +> + char **tmppath, char **newpath, char **newdir) +> +{ +> + pid_t pid; +> + char hostname[256]; +> + struct timeval tv; +> + char *filename; +> + int fd =3D -1; +> + +> + /* We follow the Dovecot file name generation algorithm. */ +> + pid =3D getpid (); +> + safe_gethostname (hostname, sizeof (hostname)); +> + do { +> + gettimeofday (&tv, NULL); +> + filename =3D talloc_asprintf (ctx, "%ld.M%ldP%d.%s", +> + tv.tv_sec, tv.tv_usec, pid, hostname); +> + if (! filename) { +> + fprintf (stderr, "Out of memory\n"); +> + return -1; +> + } +> + +> + *tmppath =3D talloc_asprintf (ctx, "%s/tmp/%s", dir, filename); +> + if (! *tmppath) { +> + fprintf (stderr, "Out of memory\n"); +> + return -1; +> + } +> + +> + fd =3D open (*tmppath, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600); +> + } while (fd =3D=3D -1 && errno =3D=3D EEXIST); +> + +> + if (fd =3D=3D -1) { +> + fprintf (stderr, "Error: opening %s: %s\n", *tmppath, strerror (errno)); +> + return -1; +> + } +> + +> + *newdir =3D talloc_asprintf (ctx, "%s/new", dir); +> + *newpath =3D talloc_asprintf (ctx, "%s/new/%s", dir, filename); +> + if (! *newdir || ! *newpath) { +> + fprintf (stderr, "Out of memory\n"); +> + close (fd); +> + unlink (*tmppath); +> + return -1; +> + } +> + +> + talloc_free (filename); +> + +> + return fd; +> +} +> + +> +/* Copy the contents of standard input (fdin) into fdout. +> + * Returns TRUE if a non-empty file was written successfully. +> + * Otherwise, return FALSE. */ +> +static notmuch_bool_t +> +copy_stdin (int fdin, int fdout) +> +{ +> + notmuch_bool_t empty =3D TRUE; +> + +> + while (! interrupted) { +> + ssize_t remain; +> + char buf[4096]; +> + char *p; +> + +> + remain =3D read (fdin, buf, sizeof (buf)); +> + if (remain =3D=3D 0) +> + break; +> + if (remain < 0) { +> + if (errno =3D=3D EINTR) +> + continue; +> + fprintf (stderr, "Error: reading from standard input: %s\n", +> + strerror (errno)); +> + return FALSE; +> + } +> + +> + p =3D buf; +> + do { +> + ssize_t written =3D write (fdout, p, remain); +> + if (written < 0 && errno =3D=3D EINTR) +> + continue; +> + if (written <=3D 0) { +> + fprintf (stderr, "Error: writing to temporary file: %s", +> + strerror (errno)); +> + return FALSE; +> + } +> + p +=3D written; +> + remain -=3D written; +> + empty =3D FALSE; +> + } while (remain > 0); +> + } +> + +> + return (!interrupted && !empty); +> +} +> + +> +/* Add the specified message file to the notmuch database, applying tags. +> + * The file is renamed to encode notmuch tags as maildir flags. */ +> +static void +> +add_file_to_database (notmuch_database_t *notmuch, const char *path, +> + tag_op_list_t *tag_ops) +> +{ + +I think I would like the option to have the command fail if it doesn't +get added to the database. If, for example, the command is used to +postpone a message it may be very difficult/impossible to find it +without the postponed tag that I asked for. + +Could we have a --fail-on-index-fail cli option which says which +behaviour we want? It is a little ugly but I think different users will +want different behaviour and there is a risk of dataloss in either case. + +Best wishes + +Mark + + + +> + notmuch_message_t *message; +> + notmuch_status_t status; +> + +> + status =3D notmuch_database_add_message (notmuch, path, &message); +> + switch (status) { +> + case NOTMUCH_STATUS_SUCCESS: +> + case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: +> + break; +> + default: +> + case NOTMUCH_STATUS_FILE_NOT_EMAIL: +> + case NOTMUCH_STATUS_READ_ONLY_DATABASE: +> + case NOTMUCH_STATUS_XAPIAN_EXCEPTION: +> + case NOTMUCH_STATUS_OUT_OF_MEMORY: +> + case NOTMUCH_STATUS_FILE_ERROR: +> + case NOTMUCH_STATUS_NULL_POINTER: +> + case NOTMUCH_STATUS_TAG_TOO_LONG: +> + case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: +> + case NOTMUCH_STATUS_UNBALANCED_ATOMIC: +> + case NOTMUCH_STATUS_LAST_STATUS: +> + fprintf (stderr, "Error: failed to add `%s' to notmuch database: %s\n", +> + path, notmuch_status_to_string (status)); +> + return; +> + } +> + +> + if (status =3D=3D NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID) { +> + /* Don't change tags of an existing message. */ +> + status =3D notmuch_message_tags_to_maildir_flags (message); +> + if (status !=3D NOTMUCH_STATUS_SUCCESS) +> + fprintf (stderr, "Error: failed to sync tags to maildir flags\n"); +> + } else { +> + tag_op_list_apply (message, tag_ops, TAG_FLAG_MAILDIR_SYNC); +> + } +> + +> + notmuch_message_destroy (message); +> +} +> + +> +static notmuch_bool_t +> +insert_message (void *ctx, notmuch_database_t *notmuch, int fdin, +> + const char *dir, tag_op_list_t *tag_ops) +> +{ +> + char *tmppath; +> + char *newpath; +> + char *newdir; +> + int fdout; +> + char *cleanup_path; +> + +> + fdout =3D maildir_open_tmp_file (ctx, dir, &tmppath, &newpath, &newd= +ir); +> + if (fdout < 0) +> + return FALSE; +> + +> + cleanup_path =3D tmppath; +> + +> + if (! copy_stdin (fdin, fdout)) +> + goto FAIL; +> + +> + if (fsync (fdout) !=3D 0) { +> + fprintf (stderr, "Error: fsync failed: %s\n", strerror (errno)); +> + goto FAIL; +> + } +> + +> + close (fdout); +> + fdout =3D -1; +> + +> + /* Atomically move the new message file from the Maildir 'tmp' direc= +tory +> + * to the 'new' directory. We follow the Dovecot recommendation to +> + * simply use rename() instead of link() and unlink(). +> + * See also: http://wiki.dovecot.org/MailboxFormat/Maildir#Mail_deli= +very +> + */ +> + if (rename (tmppath, newpath) !=3D 0) { +> + fprintf (stderr, "Error: rename() failed: %s\n", strerror (errno)); +> + goto FAIL; +> + } +> + +> + if (! sync_dir (newdir)) +> + goto FAIL; +> + +> + /* Even if adding the message to the notmuch database fails, +> + * the message is on disk and we consider the delivery completed. */ +> + add_file_to_database (notmuch, newpath, tag_ops); +> + +> + return TRUE; +> + +> + FAIL: +> + if (fdout >=3D 0) +> + close (fdout); +> + unlink (cleanup_path); +> + return FALSE; +> +} +> + +> +int +> +notmuch_insert_command (notmuch_config_t *config, int argc, char *argv[]) +> +{ +> + notmuch_database_t *notmuch; +> + struct sigaction action; +> + const char *db_path; +> + const char **new_tags; +> + size_t new_tags_length; +> + tag_op_list_t *tag_ops; +> + char *query_string =3D NULL; +> + const char *maildir; +> + int opt_index =3D 1; +> + unsigned int i; +> + notmuch_bool_t ret; +> + +> + db_path =3D notmuch_config_get_database_path (config); +> + new_tags =3D notmuch_config_get_new_tags (config, &new_tags_length); +> + +> + tag_ops =3D tag_op_list_create (config); +> + if (tag_ops =3D=3D NULL) { +> + fprintf (stderr, "Out of memory.\n"); +> + return 1; +> + } +> + for (i =3D 0; i < new_tags_length; i++) { +> + if (tag_op_list_append (tag_ops, new_tags[i], FALSE)) +> + return 1; +> + } +> + +> + if (parse_tag_command_line (config, argc - opt_index, argv + opt_ind= +ex, +> + &query_string, tag_ops)) +> + return 1; +> + +> + if (*query_string !=3D '\0') { +> + fprintf (stderr, "Error: unexpected query string: %s\n", query_string); +> + return 1; +> + } +> + +> + maildir =3D db_path; +> + +> + /* Setup our handler for SIGINT. We do not set SA_RESTART so that co= +pying +> + * from standard input may be interrupted. */ +> + memset (&action, 0, sizeof (struct sigaction)); +> + action.sa_handler =3D handle_sigint; +> + sigemptyset (&action.sa_mask); +> + action.sa_flags =3D 0; +> + sigaction (SIGINT, &action, NULL); +> + +> + if (notmuch_database_open (notmuch_config_get_database_path (config), +> + NOTMUCH_DATABASE_MODE_READ_WRITE, ¬much)) +> + return 1; +> + +> + ret =3D insert_message (config, notmuch, STDIN_FILENO, maildir, tag_= +ops); +> + +> + notmuch_database_destroy (notmuch); +> + +> + return (ret) ? 0 : 1; +> +} +> diff --git a/notmuch.c b/notmuch.c +> index 45a73ce..cf5bf95 100644 +> --- a/notmuch.c +> +++ b/notmuch.c +> @@ -44,6 +44,9 @@ static command_t commands[] =3D { +> "Interactively setup notmuch for first use." }, +> { "new", notmuch_new_command, FALSE, +> "Find and import new messages to the notmuch database." }, +> + { "insert", notmuch_insert_command, FALSE, +> + "[options...] [--] [+|- ...] < message", +> + "Add a new message into the maildir and notmuch database." }, +> { "search", notmuch_search_command, FALSE, +> "Search for messages matching the given search terms." }, +> { "show", notmuch_show_command, FALSE, +> --=20 +> 1.7.12.1 +> +> _______________________________________________ +> notmuch mailing list +> notmuch@notmuchmail.org +> http://notmuchmail.org/mailman/listinfo/notmuch -- 2.26.2