stream filter: add "no more input" to the filters

Some filters may need to buffer the input and look-ahead inside it
to decide what to output, and they may consume more than zero bytes
of input and still not produce any output. After feeding all the
input, pass NULL as input as keep calling stream_filter() to let
such filters know there is no more input coming, and it is time for
them to produce the remaining output based on the buffered input.

Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/convert.c b/convert.c
index 1ec91a370e98c9bcaa2d539935ae45005b9f7d68..4951372db821f02db512445602055c68a7c9309b 100644 (file)
--- a/convert.c
+++ b/convert.c
@@ -838,7 +838,11 @@ static int null_filter_fn(struct stream_filter *filter,
                          const char *input, size_t *isize_p,
                          char *output, size_t *osize_p)
 {
-       size_t count = *isize_p;
+       size_t count;
+
+       if (!input)
+               return 0; /* we do not keep any states */
+       count = *isize_p;
        if (*osize_p < count)
                count = *osize_p;
        if (count) {

diff --git a/convert.h b/convert.h
index 17d75098328c65638c435d451564999db3cad281..d799a165b4731f2a44a44eaceffc588d1e9fcfc3 100644 (file)
--- a/convert.h
+++ b/convert.h
@@ -57,6 +57,13 @@ extern int is_null_stream_filter(struct stream_filter *);
  * Use as much input up to *isize_p and fill output up to *osize_p;
  * update isize_p and osize_p to indicate how much buffer space was
  * consumed and filled. Return 0 on success, non-zero on error.
+ *
+ * Some filters may need to buffer the input and look-ahead inside it
+ * to decide what to output, and they may consume more than zero bytes
+ * of input and still not produce any output. After feeding all the
+ * input, pass NULL as input and keep calling this function, to let
+ * such filters know there is no more input coming and it is time for
+ * them to produce the remaining output based on the buffered input.
  */
 extern int stream_filter(struct stream_filter *,
                         const char *input, size_t *isize_p,

diff --git a/streaming.c b/streaming.c
index 565f000790b482a8977a36ca9a9cfcbd633f056d..91414f459231a83e6d22ee5a91bd0f6bc65ff5b0 100644 (file)
--- a/streaming.c
+++ b/streaming.c
@@ -60,6 +60,7 @@ struct filtered_istream {
        char obuf[FILTER_BUFFER];
        int i_end, i_ptr;
        int o_end, o_ptr;
+       int input_finished;
 };
 
 struct git_istream {
@@ -215,12 +216,30 @@ static read_method_decl(filtered)
                        fs->o_end = FILTER_BUFFER - to_receive;
                        continue;
                }
+
+               /* tell the filter to drain upon no more input */
+               if (fs->input_finished) {
+                       size_t to_receive = FILTER_BUFFER;
+                       if (stream_filter(fs->filter,
+                                         NULL, NULL,
+                                         fs->obuf, &to_receive))
+                               return -1;
+                       fs->o_end = FILTER_BUFFER - to_receive;
+                       if (!fs->o_end)
+                               break;
+                       continue;
+               }
                fs->i_end = fs->i_ptr = 0;
 
                /* refill the input from the upstream */
-               fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER);
-               if (fs->i_end <= 0)
-                       break;
+               if (!fs->input_finished) {
+                       fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER);
+                       if (fs->i_end < 0)
+                               break;
+                       if (fs->i_end)
+                               continue;
+               }
+               fs->input_finished = 1;
        }
        return filled;
 }
@@ -241,6 +260,7 @@ static struct git_istream *attach_stream_filter(struct git_istream *st,
        fs->filter = filter;
        fs->i_end = fs->i_ptr = 0;
        fs->o_end = fs->o_ptr = 0;
+       fs->input_finished = 0;
        ifs->size = -1; /* unknown */
        return ifs;
 }