From b3ce9852c53d456fd9c7a323c0fe1880cdb82ee2 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Tue, 22 Jan 2013 23:45:43 +0200 Subject: [PATCH] Re: [PATCH v3 01/20] cli: add stub for insert command --- 48/fd4cb04bb77df2feb3df9be98dac017e3cfd3b | 543 ++++++++++++++++++++++ 1 file changed, 543 insertions(+) create mode 100644 48/fd4cb04bb77df2feb3df9be98dac017e3cfd3b diff --git a/48/fd4cb04bb77df2feb3df9be98dac017e3cfd3b b/48/fd4cb04bb77df2feb3df9be98dac017e3cfd3b new file mode 100644 index 000000000..dd4ff9729 --- /dev/null +++ b/48/fd4cb04bb77df2feb3df9be98dac017e3cfd3b @@ -0,0 +1,543 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 6FA95431FB6 + for ; Tue, 22 Jan 2013 13:45:50 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id UtWJbfZMCSWn for ; + Tue, 22 Jan 2013 13:45:49 -0800 (PST) +Received: from mail-la0-f54.google.com (mail-la0-f54.google.com + [209.85.215.54]) (using TLSv1 with cipher RC4-SHA (128/128 bits)) + (No client certificate requested) + by olra.theworths.org (Postfix) with ESMTPS id 944B2431FAF + for ; Tue, 22 Jan 2013 13:45:48 -0800 (PST) +Received: by mail-la0-f54.google.com with SMTP id gw10so7085995lab.41 + for ; Tue, 22 Jan 2013 13:45:47 -0800 (PST) +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=google.com; s=20120113; + h=x-received:from:to:subject:in-reply-to:references:user-agent:date + :message-id:mime-version:content-type:content-transfer-encoding + :x-gm-message-state; + bh=+O/d8PJKF9Oye9+3pmLLre214k7HdRAgWA9ZxFTcfZs=; + b=X68Xuy0zoUo8wecnYSBIVZQwL7EwEYAnVxJDQsXKsh8fdC3DwneFcgSJljHsawW5AF + aQKRDMJs+t1koQ/AAjbQZOAJu1LAG1UetBgViuqmv4zNi20PJKdE1o+obrtxjiMBY1c5 + /T2hUe7FY+ZKgOSfYb2/4SQMjWzergDBL0K28vrJERXGCZj0kHxSUq5gMTnWSqZ7ygdp + AdHeEvvYQjPhVO4SYdmCb7Vl+WWOwZbzlmSLBFDKKBNlupTmkcGe4aDCv6fDbyLVvqBZ + KINQi3+v0lIfG+YCzW6ns9qJqPMkKdlnPZM87J8fbZmImQ4wi77P8d2yCAyAkQbDvFeT + Y5sg== +X-Received: by 10.112.23.34 with SMTP id j2mr9813037lbf.118.1358891145572; + Tue, 22 Jan 2013 13:45:45 -0800 (PST) +Received: from localhost (dsl-hkibrasgw4-50df51-27.dhcp.inet.fi. + [80.223.81.27]) + by mx.google.com with ESMTPS id ft8sm7511229lab.9.2013.01.22.13.45.43 + (version=TLSv1.2 cipher=RC4-SHA bits=128/128); + Tue, 22 Jan 2013 13:45:44 -0800 (PST) +From: Jani Nikula +To: Peter Wang , notmuch@notmuchmail.org +Subject: Re: [PATCH v3 01/20] cli: add stub for insert command +In-Reply-To: <1358643004-14522-2-git-send-email-novalazy@gmail.com> +References: <1358643004-14522-1-git-send-email-novalazy@gmail.com> + <1358643004-14522-2-git-send-email-novalazy@gmail.com> +User-Agent: Notmuch/0.14+255~gff3cc55 (http://notmuchmail.org) Emacs/24.2.1 + (x86_64-pc-linux-gnu) +Date: Tue, 22 Jan 2013 23:45:43 +0200 +Message-ID: <87vcaoj3i0.fsf@nikula.org> +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: quoted-printable +X-Gm-Message-State: + ALoCoQkJd7fmbnhlqI0o+eYsHjLUH6tK3udqP0ZQgTwJ+I38rbLOueoL52URBy6v2sC4V4vjLHS8 +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Tue, 22 Jan 2013 21:45:50 -0000 + + +Hi Peter - + +On Sun, 20 Jan 2013, Peter Wang wrote: +> The notmuch insert command should read a message from standard input +> and deliver it to a Maildir folder, and then incorporate the message +> into the notmuch database. Essentially it moves the functionality of +> notmuch-deliver into notmuch. + +Bikeshedding first, I'd prefer "notmuch deliver" for the command too... + +> Though it could be used as an alternative to notmuch new, the reason +> I want this is to allow my notmuch frontend to add postponed or sent +> messages to the mail store and notmuch database, without resorting to +> another tool (e.g. notmuch-deliver) nor directly modifying the maildir. + +This review is based on the following patches squashed together: + + cli: add stub for insert command + insert: open Maildir tmp file + insert: copy stdin to Maildir tmp file + insert: move file from Maildir tmp to new + insert: add new message to database + insert: apply default tags to new message + insert: parse and apply command-line tag operations + insert: fsync after writing tmp file + insert: trap SIGINT and clean up + insert: add copyright line from notmuch-deliver + +It's much easier for me to grasp the big picture this way. + +> --- +> Makefile.local | 1 + +> notmuch-client.h | 3 + +> notmuch-insert.c | 316 ++++++++++++++++++++++++++++++++++++++++++++++++= +++++++ +> notmuch.c | 3 + +> 4 files changed, 323 insertions(+) +> create mode 100644 notmuch-insert.c +> +> diff --git a/Makefile.local b/Makefile.local +> index c274f07..bb2381d 100644 +> --- a/Makefile.local +> +++ b/Makefile.local +> @@ -261,6 +261,7 @@ notmuch_client_srcs =3D \ +> notmuch-config.c \ +> notmuch-count.c \ +> notmuch-dump.c \ +> + notmuch-insert.c \ +> notmuch-new.c \ +> notmuch-reply.c \ +> notmuch-restore.c \ +> diff --git a/notmuch-client.h b/notmuch-client.h +> index 5f28836..af7d094 100644 +> --- a/notmuch-client.h +> +++ b/notmuch-client.h +> @@ -175,6 +175,9 @@ int +> notmuch_dump_command (void *ctx, int argc, char *argv[]); +>=20=20 +> int +> +notmuch_insert_command (void *ctx, int argc, char *argv[]); +> + +> +int +> notmuch_new_command (void *ctx, int argc, char *argv[]); +>=20=20 +> int +> diff --git a/notmuch-insert.c b/notmuch-insert.c +> new file mode 100644 +> index 0000000..0e74be0 +> --- /dev/null +> +++ b/notmuch-insert.c +> @@ -0,0 +1,316 @@ +> +/* notmuch - Not much of an email program, (just index and search) +> + * +> + * Copyright =C2=A9 2013 Peter Wang +> + * +> + * Based in part on notmuch-deliver +> + * Copyright =C2=A9 2010 Ali Polatel +> + * +> + * This program is free software: you can redistribute it and/or modify +> + * it under the terms of the GNU General Public License as published by +> + * the Free Software Foundation, either version 3 of the License, or +> + * (at your option) any later version. +> + * +> + * This program is distributed in the hope that it will be useful, +> + * but WITHOUT ANY WARRANTY; without even the implied warranty of +> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +> + * GNU General Public License for more details. +> + * +> + * You should have received a copy of the GNU General Public License +> + * along with this program. If not, see http://www.gnu.org/licenses/ . +> + * +> + * Author: Peter Wang +> + */ +> + +> +#include "notmuch-client.h" +> +#include "tag-util.h" +> + +> +#include +> +#include +> +#include +> + +> +static volatile sig_atomic_t interrupted; +> + +> +static void +> +handle_sigint (unused (int sig)) +> +{ +> + static char msg[] =3D "Stopping... \n"; +> + +> + /* This write is "opportunistic", so it's okay to ignore the +> + * result. It is not required for correctness, and if it does +> + * fail or produce a short write, we want to get out of the signal +> + * handler as quickly as possible, not retry it. */ +> + IGNORE_RESULT (write (2, msg, sizeof (msg) - 1)); +> + interrupted =3D 1; +> +} +> + +> +/* Like gethostname but guarantees that a null-terminated hostname is +> + * returned, even if it has to make one up. +> + * Returns true unless hostname contains a slash. */ +> +static notmuch_bool_t +> +safe_gethostname (char *hostname, size_t len) +> +{ +> + if (gethostname (hostname, len) =3D=3D -1) { +> + strncpy (hostname, "unknown", len); +> + } +> + hostname[len - 1] =3D '\0'; +> + +> + return (strchr (hostname, '/') =3D=3D NULL); + +You could just replace all chars you don't accept with something you +do. Add ':' to the list of unacceptable chars. + +> +} +> + +> +/* Open a unique file in the Maildir 'tmp' directory. +> + * Returns the file descriptor on success, or -1 on failure. +> + * On success, file paths for the message in the 'tmp' and 'new' +> + * directories are returned via tmppath and newpath. */ +> +static int +> +maildir_open_tmp_file (void *ctx, const char *dir, +> + char **tmppath, char **newpath) +> +{ +> + pid_t pid; +> + char hostname[256]; +> + struct timeval tv; +> + char *filename; +> + int fd =3D -1; +> + +> + /* We follow the Dovecot file name generation algorithm. */ + +See also http://cr.yp.to/proto/maildir.html + +> + pid =3D getpid (); +> + if (! safe_gethostname (hostname, sizeof (hostname))) { +> + fprintf (stderr, "Error: invalid host name.\n"); +> + return -1; +> + } +> + do { +> + gettimeofday (&tv, NULL); +> + filename =3D talloc_asprintf (ctx, "%ld.M%ldP%d.%s", +> + tv.tv_sec, tv.tv_usec, pid, hostname); +> + if (! filename) { +> + fprintf (stderr, "Out of memory\n"); +> + return -1; +> + } +> + +> + *tmppath =3D talloc_asprintf (ctx, "%s/tmp/%s", dir, filename); +> + if (! *tmppath) { +> + fprintf (stderr, "Out of memory\n"); +> + return -1; +> + } +> + +> + fd =3D open (*tmppath, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, 0600); +> + } while (fd =3D=3D -1 && errno =3D=3D EEXIST); +> + +> + if (fd =3D=3D -1) { +> + fprintf (stderr, "Error: opening %s: %s\n", *tmppath, strerror (errno)); +> + return -1; +> + } +> + +> + *newpath =3D talloc_asprintf (ctx, "%s/new/%s", dir, filename); +> + if (! *newpath) { +> + fprintf (stderr, "Out of memory\n"); +> + close (fd); +> + unlink (*tmppath); +> + return -1; +> + } +> + +> + talloc_free (filename); + +Nitpick, in theory the do-while loop above could allocate a bunch of +filenames and paths that you do not free (they're freed as part of the +context). + +> + +> + return fd; +> +} +> + +> +/* Atomically move the new message file from the Maildir 'tmp' directory +> + * to the 'new' directory. +> + * +> + * We follow the Dovecot recommendation to simply use rename() +> + * instead of link() and unlink(). See also: +> + * http://wiki.dovecot.org/MailboxFormat/Maildir#Mail_delivery +> + */ +> +static notmuch_bool_t +> +maildir_move_tmp_to_new (const char *tmppath, const char *newpath) +> +{ +> + if (rename (tmppath, newpath) !=3D 0) { +> + fprintf (stderr, "Error: rename() failed: %s\n", strerror (errno)); +> + return FALSE; +> + } +> + +> + return TRUE; +> +} + +IMO you could just use rename() inline in the caller, without a wrapper. + +> + +> +/* Copy the contents of standard input (fdin) into fdout. */ +> +static notmuch_bool_t +> +copy_stdin (int fdin, int fdout) + +The comment and the function name imply the function has something to do +with stdin, while it only cares about file descriptors. + +> +{ +> + char buf[4096]; +> + char *p; +> + ssize_t remain; +> + ssize_t written; +> + +> + while (! interrupted) { +> + remain =3D read (fdin, buf, sizeof (buf)); +> + if (remain =3D=3D 0) +> + break; +> + if (remain < 0) { +> + if (errno =3D=3D EINTR) +> + continue; +> + fprintf (stderr, "Error: reading from standard input: %s\n", +> + strerror (errno)); +> + return FALSE; +> + } +> + +> + p =3D buf; +> + do { +> + written =3D write (fdout, p, remain); +> + if (written =3D=3D 0) +> + return FALSE; + +No error message? + +> + if (written < 0) { +> + if (errno =3D=3D EINTR) +> + continue; +> + fprintf (stderr, "Error: writing to temporary file: %s", +> + strerror (errno)); +> + return FALSE; +> + } +> + p +=3D written; +> + remain -=3D written; +> + } while (remain > 0); +> + } +> + +> + return ! interrupted; +> +} +> + +> +/* Add the specified message file to the notmuch database, applying tags. +> + * The file is renamed to encode notmuch tags as maildir flags. */ +> +static notmuch_bool_t +> +add_file_to_database (notmuch_database_t *notmuch, const char *path, +> + tag_op_list_t *tag_ops) +> +{ +> + notmuch_message_t *message; +> + notmuch_status_t status; +> + +> + status =3D notmuch_database_add_message (notmuch, path, &message); +> + switch (status) { +> + case NOTMUCH_STATUS_SUCCESS: +> + break; +> + case NOTMUCH_STATUS_DUPLICATE_MESSAGE_ID: +> + fprintf (stderr, "Warning: duplicate message.\n"); + +This is not uncommon. Why the warning? + +Also, notmuch new does not apply new.tags in this case. Are you sure we +want to do that here? (You get mail, you read and archive it, you get +the dupe, it pops up unread in your inbox again.) + +> + break; +> + default: +> + case NOTMUCH_STATUS_FILE_NOT_EMAIL: +> + case NOTMUCH_STATUS_READ_ONLY_DATABASE: +> + case NOTMUCH_STATUS_XAPIAN_EXCEPTION: +> + case NOTMUCH_STATUS_OUT_OF_MEMORY: +> + case NOTMUCH_STATUS_FILE_ERROR: +> + case NOTMUCH_STATUS_NULL_POINTER: +> + case NOTMUCH_STATUS_TAG_TOO_LONG: +> + case NOTMUCH_STATUS_UNBALANCED_FREEZE_THAW: +> + case NOTMUCH_STATUS_UNBALANCED_ATOMIC: +> + case NOTMUCH_STATUS_LAST_STATUS: +> + fprintf (stderr, "Error: failed to add `%s' to notmuch database: %s\n", +> + path, notmuch_status_to_string (status)); +> + return FALSE; +> + } +> + +> + tag_op_list_apply (message, tag_ops, TAG_FLAG_MAILDIR_SYNC); + +Check return value. + +> + +> + notmuch_message_destroy (message); +> + +> + return TRUE; +> +} +> + +> +static notmuch_bool_t +> +insert_message (void *ctx, notmuch_database_t *notmuch, int fdin, +> + const char *dir, tag_op_list_t *tag_ops) +> +{ +> + char *tmppath; +> + char *newpath; +> + int fdout; +> + notmuch_bool_t ret; +> + +> + fdout =3D maildir_open_tmp_file (ctx, dir, &tmppath, &newpath); +> + if (fdout < 0) { +> + return FALSE; +> + } +> + ret =3D copy_stdin (fdin, fdout); +> + if (ret && fsync (fdout) !=3D 0) { + +Keep ret check and fsync separate. + +On some file systems you need to fsync the directory after adding a new +file as well. + +> + fprintf (stderr, "Error: fsync failed: %s\n", strerror (errno)); +> + ret =3D FALSE; +> + } +> + close (fdout); +> + if (ret) { +> + ret =3D maildir_move_tmp_to_new (tmppath, newpath); +> + } +> + if (!ret) { +> + unlink (tmppath); +> + return FALSE; +> + } + +I think you should clean up all of the error paths above. It's not +obvious at a glance what happens. Typically it should be just blocks +like this repeated: + + ret =3D foo(); + if (ret) { + /* error handling */ + } + +> + +> + ret =3D add_file_to_database (notmuch, newpath, tag_ops); +> + if (!ret) { +> + /* XXX maybe there should be an option to keep the file in maildir? */ + +Yes, in the future. + +I might like it better if you separated writing the file to maildir and +adding the file to the database in the top level +notmuch_insert_command() function. The delivery function could return a +char * to the filename so it could be passed to +add_file_to_database(). And you wouldn't need to pass database or tag +ops to the maildir writing function, keeping things clearly separated. + +> + unlink (newpath); +> + return FALSE; +> + } +> + +> + return TRUE; +> +} +> + +> +int +> +notmuch_insert_command (void *ctx, int argc, char *argv[]) +> +{ +> + notmuch_config_t *config; +> + notmuch_database_t *notmuch; +> + struct sigaction action; +> + const char *db_path; +> + const char **new_tags; +> + size_t new_tags_length; +> + tag_op_list_t *tag_ops; +> + char *query_string =3D NULL; +> + char *maildir; +> + int opt_index =3D 1; +> + unsigned int i; +> + notmuch_bool_t ret; +> + +> + config =3D notmuch_config_open (ctx, NULL, NULL); +> + if (config =3D=3D NULL) +> + return 1; +> + +> + db_path =3D notmuch_config_get_database_path (config); +> + new_tags =3D notmuch_config_get_new_tags (config, &new_tags_length); +> + +> + tag_ops =3D tag_op_list_create (ctx); +> + if (tag_ops =3D=3D NULL) { +> + fprintf (stderr, "Out of memory.\n"); +> + return 1; +> + } +> + for (i =3D 0; i < new_tags_length; i++) { +> + if (tag_op_list_append (tag_ops, new_tags[i], FALSE)) +> + return 1; +> + } +> + +> + if (parse_tag_command_line (ctx, argc - opt_index, argv + opt_index, +> + &query_string, tag_ops)) +> + return 1; + +Never mind about my earlier comment about changing +parse_tag_command_line(). It's probably better to allow this to override +new.tags. + +> + +> + if (*query_string !=3D '\0') { +> + fprintf (stderr, "Error: unexpected query string: %s\n", query_string); +> + return 1; +> + } +> + +> + maildir =3D talloc_asprintf (ctx, "%s", db_path); +> + if (! maildir) { +> + fprintf (stderr, "Out of memory\n"); +> + return 1; +> + } + +There's no need to talloc maildir; it's the same as db_path. + +> + +> + /* Setup our handler for SIGINT. We do not set SA_RESTART so that co= +pying +> + * from standard input may be interrupted. */ +> + memset (&action, 0, sizeof (struct sigaction)); +> + action.sa_handler =3D handle_sigint; +> + sigemptyset (&action.sa_mask); +> + action.sa_flags =3D 0; +> + sigaction (SIGINT, &action, NULL); +> + +> + if (notmuch_database_open (notmuch_config_get_database_path (config), +> + NOTMUCH_DATABASE_MODE_READ_WRITE, ¬much)) +> + return 1; +> + +> + ret =3D insert_message (ctx, notmuch, STDIN_FILENO, maildir, tag_ops= +); +> + +> + notmuch_database_destroy (notmuch); +> + +> + return (ret) ? 0 : 1; +> +} +> diff --git a/notmuch.c b/notmuch.c +> index 4fc0973..1c3b893 100644 +> --- a/notmuch.c +> +++ b/notmuch.c +> @@ -53,6 +53,9 @@ static command_t commands[] =3D { +> { "new", notmuch_new_command, +> "[options...]", +> "Find and import new messages to the notmuch database." }, +> + { "insert", notmuch_insert_command, +> + "[options...] [--] [+|- ...] < message", +> + "Add a new message into the maildir and notmuch database." }, +> { "search", notmuch_search_command, +> "[options...] [...]", +> "Search for messages matching the given search terms." }, +> --=20 +> 1.7.10.4 -- 2.26.2