From bd8eb43bd63c9d6a111f0ac04dd609098492617d Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sun, 25 Nov 2012 12:41:34 +1900 Subject: [PATCH] Re: [PATCH v2 1/7] cli: allow query to come from stdin --- 2d/e8fd29800f68884cc4d5a78ab594ab2a1d04aa | 213 ++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 2d/e8fd29800f68884cc4d5a78ab594ab2a1d04aa diff --git a/2d/e8fd29800f68884cc4d5a78ab594ab2a1d04aa b/2d/e8fd29800f68884cc4d5a78ab594ab2a1d04aa new file mode 100644 index 000000000..a168c3dc3 --- /dev/null +++ b/2d/e8fd29800f68884cc4d5a78ab594ab2a1d04aa @@ -0,0 +1,213 @@ +Return-Path: +X-Original-To: notmuch@notmuchmail.org +Delivered-To: notmuch@notmuchmail.org +Received: from localhost (localhost [127.0.0.1]) + by olra.theworths.org (Postfix) with ESMTP id 782B8431FAF + for ; Sat, 24 Nov 2012 09:41:38 -0800 (PST) +X-Virus-Scanned: Debian amavisd-new at olra.theworths.org +X-Spam-Flag: NO +X-Spam-Score: -0.7 +X-Spam-Level: +X-Spam-Status: No, score=-0.7 tagged_above=-999 required=5 + tests=[RCVD_IN_DNSWL_LOW=-0.7] autolearn=disabled +Received: from olra.theworths.org ([127.0.0.1]) + by localhost (olra.theworths.org [127.0.0.1]) (amavisd-new, port 10024) + with ESMTP id r+KOVW0xgucj for ; + Sat, 24 Nov 2012 09:41:37 -0800 (PST) +Received: from dmz-mailsec-scanner-6.mit.edu (DMZ-MAILSEC-SCANNER-6.MIT.EDU + [18.7.68.35]) + by olra.theworths.org (Postfix) with ESMTP id C13E2431FAE + for ; Sat, 24 Nov 2012 09:41:37 -0800 (PST) +X-AuditID: 12074423-b7fab6d0000008f9-d6-50b106d19812 +Received: from mailhub-auth-4.mit.edu ( [18.7.62.39]) + by dmz-mailsec-scanner-6.mit.edu (Symantec Messaging Gateway) with SMTP + id 0A.F2.02297.1D601B05; Sat, 24 Nov 2012 12:41:37 -0500 (EST) +Received: from outgoing.mit.edu (OUTGOING-AUTH.MIT.EDU [18.7.22.103]) + by mailhub-auth-4.mit.edu (8.13.8/8.9.2) with ESMTP id qAOHfaN0029112; + Sat, 24 Nov 2012 12:41:37 -0500 +Received: from awakening.csail.mit.edu (awakening.csail.mit.edu [18.26.4.91]) + (authenticated bits=0) + (User authenticated as amdragon@ATHENA.MIT.EDU) + by outgoing.mit.edu (8.13.6/8.12.4) with ESMTP id qAOHfYxm012063 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-SHA bits=128 verify=NOT); + Sat, 24 Nov 2012 12:41:35 -0500 (EST) +Received: from amthrax by awakening.csail.mit.edu with local (Exim 4.80) + (envelope-from ) + id 1TcJjS-0004gV-Fc; Sat, 24 Nov 2012 12:41:34 -0500 +Date: Sat, 24 Nov 2012 12:41:34 -0500 +From: Austin Clements +To: markwalters1009 +Subject: Re: [PATCH v2 1/7] cli: allow query to come from stdin +Message-ID: <20121124174134.GH4562@mit.edu> +References: <1353763256-32336-1-git-send-email-markwalters1009@gmail.com> + <1353763256-32336-2-git-send-email-markwalters1009@gmail.com> +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline +In-Reply-To: <1353763256-32336-2-git-send-email-markwalters1009@gmail.com> +User-Agent: Mutt/1.5.21 (2010-09-15) +X-Brightmail-Tracker: + H4sIAAAAAAAAA+NgFmpmleLIzCtJLcpLzFFi42IRYrdT173ItjHA4NxxTovVc3ksrt+cyezA + 5LFz1l12j2erbjEHMEVx2aSk5mSWpRbp2yVwZex5/Jy14IFMxeUdE1kaGOeLdjFycEgImEgc + WyDVxcgJZIpJXLi3nq2LkYtDSGAfo0TTha3MEM4GRom1R/ugnItMEnfn3mGHcJYwSvz6OJcd + pJ9FQFXixor1zCA2m4CGxLb9yxlBbBEBfYk9K26zgdjMAtIS3343M4HYwgL2Eq0f2llAbF4B + bYk9u3ZB7e5klOi+vZsJIiEocXLmExaIZi2JG/9eMoHcDTJo+T8OkDCngJfEuRO/wXaJCqhI + TDm5jW0Co9AsJN2zkHTPQuhewMi8ilE2JbdKNzcxM6c4NVm3ODkxLy+1SNdMLzezRC81pXQT + IzisXZR3MP45qHSIUYCDUYmHN+HM+gAh1sSy4srcQ4ySHExKorwlrBsDhPiS8lMqMxKLM+KL + SnNSiw8xSnAwK4nwPjuyIUCINyWxsiq1KB8mJc3BoiTOey3lpr+QQHpiSWp2ampBahFMVoaD + Q0mC9z/IUMGi1PTUirTMnBKENBMHJ8hwHqDhOSA1vMUFibnFmekQ+VOMilLivEdBEgIgiYzS + PLheWNp5xSgO9IowLw8wCQnxAFMWXPcroMFMQIOfzl4HMrgkESEl1cAoUd/dwqlaeKs372RC + dhLDvXsrzQ8+/LLOymvlO995B3huhLd/O/Lg4dKM+z+2Hz8UIq88e6XBQrll380O+C8XKeOW + CFj/VfXk+rLn351PR9T5zzPna1u9pi55jfKFHOcLuhMPFD63nfZ6su0nhrQr76IUHje6iWt1 + mczb/teqv26tTcLXpv1hSizFGYmGWsxFxYkAhxI1AhYDAAA= +Cc: notmuch@notmuchmail.org +X-BeenThere: notmuch@notmuchmail.org +X-Mailman-Version: 2.1.13 +Precedence: list +List-Id: "Use and development of the notmuch mail system." + +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +X-List-Received-Date: Sat, 24 Nov 2012 17:41:38 -0000 + +Quoth markwalters1009 on Nov 24 at 1:20 pm: +> From: Mark Walters +> +> After this series there will be times when a caller will want to pass +> a very large query string to notmuch (eg a list of 10,000 message-ids) +> and this can exceed the size of ARG_MAX. Hence allow notmuch to take +> the query from stdin (if the query is -). +> --- +> query-string.c | 41 +++++++++++++++++++++++++++++++++++++++++ +> 1 files changed, 41 insertions(+), 0 deletions(-) +> +> diff --git a/query-string.c b/query-string.c +> index 6536512..b1fbdeb 100644 +> --- a/query-string.c +> +++ b/query-string.c +> @@ -20,6 +20,44 @@ +> +> #include "notmuch-client.h" +> +> +/* Read a single query string from STDIN, using +> + * 'ctx' as the talloc owner for all allocations. +> + * +> + * This function returns NULL in case of insufficient memory or read +> + * errors. +> + */ +> +static char * +> +query_string_from_stdin (void *ctx) +> +{ +> + char *query_string; +> + char buf[4096]; +> + ssize_t remain; +> + +> + query_string = talloc_strdup (ctx, ""); +> + if (query_string == NULL) +> + return NULL; +> + +> + for (;;) { +> + remain = read (STDIN_FILENO, buf, sizeof(buf) - 1); +> + if (remain == 0) +> + break; +> + if (remain < 0) { +> + if (errno == EINTR) +> + continue; +> + fprintf (stderr, "Error: reading from standard input: %s\n", +> + strerror (errno)); + +talloc_free (query_string) ? + +> + return NULL; +> + } +> + +> + buf[remain] = '\0'; +> + query_string = talloc_strdup_append (query_string, buf); + +Eliminate the NUL in buf and instead + talloc_strndup_append (query_string, buf, remain) ? + +Should there be some (large) bound on the size of the query string to +prevent runaway? + +> + if (query_string == NULL) + +Technically it would be good to talloc_free the old pointer here, too. + +> + return NULL; +> + } +> + +> + return query_string; +> +} +> + + +This whole approach is O(n^2), which might actually matter for large +query strings. How about (tested, but only a little): + +#define MAX_QUERY_STRING_LENGTH (16 * 1024 * 1024) + +/* Read a single query string from STDIN, using 'ctx' as the talloc + * owner for all allocations. + * + * This function returns NULL in case of insufficient memory or read + * errors. + */ +static char * +query_string_from_stdin (void *ctx) +{ + char *query_string = NULL, *new_qs; + size_t pos = 0, end = 0; + ssize_t got; + + for (;;) { + if (end - pos < 512) { + end = MAX(end * 2, 1024); + if (end >= MAX_QUERY_STRING_LENGTH) { + fprintf (stderr, "Error: query too long\n"); + goto FAIL; + } + new_qs = talloc_realloc (ctx, query_string, char, end); + if (new_qs == NULL) + goto FAIL; + query_string = new_qs; + } + + got = read (STDIN_FILENO, query_string + pos, end - pos - 1); + if (got == 0) + break; + if (got < 0) { + if (errno == EINTR) + continue; + fprintf (stderr, "Error: reading from standard input: %s\n", + strerror (errno)); + goto FAIL; + } + pos += got; + } + + query_string[pos] = '\0'; + return query_string; + + FAIL: + talloc_free (query_string); + return NULL; +} + +> /* Construct a single query string from the passed arguments, using +> * 'ctx' as the talloc owner for all allocations. +> * +> @@ -35,6 +73,9 @@ query_string_from_args (void *ctx, int argc, char *argv[]) +> char *query_string; +> int i; +> +> + if ((argc == 1) && (strcmp ("-", argv[0]) == 0)) +> + return query_string_from_stdin (ctx); +> + +> query_string = talloc_strdup (ctx, ""); +> if (query_string == NULL) +> return NULL; -- 2.26.2