builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done;
  19 static int fetch_fsck_objects = -1;
  20 static int transfer_fsck_objects = -1;
  21 static struct fetch_pack_args args = {
  22         /* .uploadpack = */ "git-upload-pack",
  23 };
  24
  25 static const char fetch_pack_usage[] =
  26 "git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
  27 "[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
  28 "[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  29
  30 #define COMPLETE        (1U << 0)
  31 #define COMMON          (1U << 1)
  32 #define COMMON_REF      (1U << 2)
  33 #define SEEN            (1U << 3)
  34 #define POPPED          (1U << 4)
  35
  36 static int marked;
  37
  38 /*
  39  * After sending this many "have"s if we do not get any new ACK , we
  40  * give up traversing our history.
  41  */
  42 #define MAX_IN_VAIN 256
  43
  44 static struct commit_list *rev_list;
  45 static int non_common_revs, multi_ack, use_sideband;
  46
  47 static void rev_list_push(struct commit *commit, int mark)
  48 {
  49         if (!(commit->object.flags & mark)) {
  50                 commit->object.flags |= mark;
  51
  52                 if (!(commit->object.parsed))
  53                         if (parse_commit(commit))
  54                                 return;
  55
  56                 commit_list_insert_by_date(commit, &rev_list);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  74 {
  75         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  76
  77         if (o && o->type == OBJ_COMMIT)
  78                 clear_commit_marks((struct commit *)o,
  79                                    COMMON | COMMON_REF | SEEN | POPPED);
  80         return 0;
  81 }
  82
  83 /*
  84    This function marks a rev and its ancestors as common.
  85    In some cases, it is desirable to mark only the ancestors (for example
  86    when only the server does not yet know that they are common).
  87 */
  88
  89 static void mark_common(struct commit *commit,
  90                 int ancestors_only, int dont_parse)
  91 {
  92         if (commit != NULL && !(commit->object.flags & COMMON)) {
  93                 struct object *o = (struct object *)commit;
  94
  95                 if (!ancestors_only)
  96                         o->flags |= COMMON;
  97
  98                 if (!(o->flags & SEEN))
  99                         rev_list_push(commit, SEEN);
 100                 else {
 101                         struct commit_list *parents;
 102
 103                         if (!ancestors_only && !(o->flags & POPPED))
 104                                 non_common_revs--;
 105                         if (!o->parsed && !dont_parse)
 106                                 if (parse_commit(commit))
 107                                         return;
 108
 109                         for (parents = commit->parents;
 110                                         parents;
 111                                         parents = parents->next)
 112                                 mark_common(parents->item, 0, dont_parse);
 113                 }
 114         }
 115 }
 116
 117 /*
 118   Get the next rev to send, ignoring the common.
 119 */
 120
 121 static const unsigned char *get_rev(void)
 122 {
 123         struct commit *commit = NULL;
 124
 125         while (commit == NULL) {
 126                 unsigned int mark;
 127                 struct commit_list *parents;
 128
 129                 if (rev_list == NULL || non_common_revs == 0)
 130                         return NULL;
 131
 132                 commit = rev_list->item;
 133                 if (!commit->object.parsed)
 134                         parse_commit(commit);
 135                 parents = commit->parents;
 136
 137                 commit->object.flags |= POPPED;
 138                 if (!(commit->object.flags & COMMON))
 139                         non_common_revs--;
 140
 141                 if (commit->object.flags & COMMON) {
 142                         /* do not send "have", and ignore ancestors */
 143                         commit = NULL;
 144                         mark = COMMON | SEEN;
 145                 } else if (commit->object.flags & COMMON_REF)
 146                         /* send "have", and ignore ancestors */
 147                         mark = COMMON | SEEN;
 148                 else
 149                         /* send "have", also for its ancestors */
 150                         mark = SEEN;
 151
 152                 while (parents) {
 153                         if (!(parents->item->object.flags & SEEN))
 154                                 rev_list_push(parents->item, mark);
 155                         if (mark & COMMON)
 156                                 mark_common(parents->item, 1, 0);
 157                         parents = parents->next;
 158                 }
 159
 160                 rev_list = rev_list->next;
 161         }
 162
 163         return commit->object.sha1;
 164 }
 165
 166 enum ack_type {
 167         NAK = 0,
 168         ACK,
 169         ACK_continue,
 170         ACK_common,
 171         ACK_ready
 172 };
 173
 174 static void consume_shallow_list(int fd)
 175 {
 176         if (args.stateless_rpc && args.depth > 0) {
 177                 /* If we sent a depth we will get back "duplicate"
 178                  * shallow and unshallow commands every time there
 179                  * is a block of have lines exchanged.
 180                  */
 181                 char line[1000];
 182                 while (packet_read_line(fd, line, sizeof(line))) {
 183                         if (!prefixcmp(line, "shallow "))
 184                                 continue;
 185                         if (!prefixcmp(line, "unshallow "))
 186                                 continue;
 187                         die("git fetch-pack: expected shallow list");
 188                 }
 189         }
 190 }
 191
 192 struct write_shallow_data {
 193         struct strbuf *out;
 194         int use_pack_protocol;
 195         int count;
 196 };
 197
 198 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 199 {
 200         struct write_shallow_data *data = cb_data;
 201         const char *hex = sha1_to_hex(graft->sha1);
 202         data->count++;
 203         if (data->use_pack_protocol)
 204                 packet_buf_write(data->out, "shallow %s", hex);
 205         else {
 206                 strbuf_addstr(data->out, hex);
 207                 strbuf_addch(data->out, '\n');
 208         }
 209         return 0;
 210 }
 211
 212 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 213 {
 214         struct write_shallow_data data;
 215         data.out = out;
 216         data.use_pack_protocol = use_pack_protocol;
 217         data.count = 0;
 218         for_each_commit_graft(write_one_shallow, &data);
 219         return data.count;
 220 }
 221
 222 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 223 {
 224         static char line[1000];
 225         int len = packet_read_line(fd, line, sizeof(line));
 226
 227         if (!len)
 228                 die("git fetch-pack: expected ACK/NAK, got EOF");
 229         if (line[len-1] == '\n')
 230                 line[--len] = 0;
 231         if (!strcmp(line, "NAK"))
 232                 return NAK;
 233         if (!prefixcmp(line, "ACK ")) {
 234                 if (!get_sha1_hex(line+4, result_sha1)) {
 235                         if (strstr(line+45, "continue"))
 236                                 return ACK_continue;
 237                         if (strstr(line+45, "common"))
 238                                 return ACK_common;
 239                         if (strstr(line+45, "ready"))
 240                                 return ACK_ready;
 241                         return ACK;
 242                 }
 243         }
 244         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 245 }
 246
 247 static void send_request(int fd, struct strbuf *buf)
 248 {
 249         if (args.stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 safe_write(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 259 }
 260
 261 #define INITIAL_FLUSH 16
 262 #define PIPESAFE_FLUSH 32
 263 #define LARGE_FLUSH 1024
 264
 265 static int next_flush(int count)
 266 {
 267         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 268
 269         if (count < flush_limit)
 270                 count <<= 1;
 271         else
 272                 count += flush_limit;
 273         return count;
 274 }
 275
 276 static int find_common(int fd[2], unsigned char *result_sha1,
 277                        struct ref *refs)
 278 {
 279         int fetching;
 280         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 281         const unsigned char *sha1;
 282         unsigned in_vain = 0;
 283         int got_continue = 0;
 284         int got_ready = 0;
 285         struct strbuf req_buf = STRBUF_INIT;
 286         size_t state_len = 0;
 287
 288         if (args.stateless_rpc && multi_ack == 1)
 289                 die("--stateless-rpc requires multi_ack_detailed");
 290         if (marked)
 291                 for_each_ref(clear_marks, NULL);
 292         marked = 1;
 293
 294         for_each_ref(rev_list_insert_ref, NULL);
 295         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 296
 297         fetching = 0;
 298         for ( ; refs ; refs = refs->next) {
 299                 unsigned char *remote = refs->old_sha1;
 300                 const char *remote_hex;
 301                 struct object *o;
 302
 303                 /*
 304                  * If that object is complete (i.e. it is an ancestor of a
 305                  * local ref), we tell them we have it but do not have to
 306                  * tell them about its ancestors, which they already know
 307                  * about.
 308                  *
 309                  * We use lookup_object here because we are only
 310                  * interested in the case we *know* the object is
 311                  * reachable and we have already scanned it.
 312                  */
 313                 if (((o = lookup_object(remote)) != NULL) &&
 314                                 (o->flags & COMPLETE)) {
 315                         continue;
 316                 }
 317
 318                 remote_hex = sha1_to_hex(remote);
 319                 if (!fetching) {
 320                         struct strbuf c = STRBUF_INIT;
 321                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 322                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 323                         if (no_done)            strbuf_addstr(&c, " no-done");
 324                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 325                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 326                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 327                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 328                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 329                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 330                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 331                         strbuf_release(&c);
 332                 } else
 333                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 334                 fetching++;
 335         }
 336
 337         if (!fetching) {
 338                 strbuf_release(&req_buf);
 339                 packet_flush(fd[1]);
 340                 return 1;
 341         }
 342
 343         if (is_repository_shallow())
 344                 write_shallow_commits(&req_buf, 1);
 345         if (args.depth > 0)
 346                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 347         packet_buf_flush(&req_buf);
 348         state_len = req_buf.len;
 349
 350         if (args.depth > 0) {
 351                 char line[1024];
 352                 unsigned char sha1[20];
 353
 354                 send_request(fd[1], &req_buf);
 355                 while (packet_read_line(fd[0], line, sizeof(line))) {
 356                         if (!prefixcmp(line, "shallow ")) {
 357                                 if (get_sha1_hex(line + 8, sha1))
 358                                         die("invalid shallow line: %s", line);
 359                                 register_shallow(sha1);
 360                                 continue;
 361                         }
 362                         if (!prefixcmp(line, "unshallow ")) {
 363                                 if (get_sha1_hex(line + 10, sha1))
 364                                         die("invalid unshallow line: %s", line);
 365                                 if (!lookup_object(sha1))
 366                                         die("object not found: %s", line);
 367                                 /* make sure that it is parsed as shallow */
 368                                 if (!parse_object(sha1))
 369                                         die("error in object: %s", line);
 370                                 if (unregister_shallow(sha1))
 371                                         die("no shallow found: %s", line);
 372                                 continue;
 373                         }
 374                         die("expected shallow/unshallow, got %s", line);
 375                 }
 376         } else if (!args.stateless_rpc)
 377                 send_request(fd[1], &req_buf);
 378
 379         if (!args.stateless_rpc) {
 380                 /* If we aren't using the stateless-rpc interface
 381                  * we don't need to retain the headers.
 382                  */
 383                 strbuf_setlen(&req_buf, 0);
 384                 state_len = 0;
 385         }
 386
 387         flushes = 0;
 388         retval = -1;
 389         while ((sha1 = get_rev())) {
 390                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 391                 if (args.verbose)
 392                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 393                 in_vain++;
 394                 if (flush_at <= ++count) {
 395                         int ack;
 396
 397                         packet_buf_flush(&req_buf);
 398                         send_request(fd[1], &req_buf);
 399                         strbuf_setlen(&req_buf, state_len);
 400                         flushes++;
 401                         flush_at = next_flush(count);
 402
 403                         /*
 404                          * We keep one window "ahead" of the other side, and
 405                          * will wait for an ACK only on the next one
 406                          */
 407                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 408                                 continue;
 409
 410                         consume_shallow_list(fd[0]);
 411                         do {
 412                                 ack = get_ack(fd[0], result_sha1);
 413                                 if (args.verbose && ack)
 414                                         fprintf(stderr, "got ack %d %s\n", ack,
 415                                                         sha1_to_hex(result_sha1));
 416                                 switch (ack) {
 417                                 case ACK:
 418                                         flushes = 0;
 419                                         multi_ack = 0;
 420                                         retval = 0;
 421                                         goto done;
 422                                 case ACK_common:
 423                                 case ACK_ready:
 424                                 case ACK_continue: {
 425                                         struct commit *commit =
 426                                                 lookup_commit(result_sha1);
 427                                         if (!commit)
 428                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 429                                         if (args.stateless_rpc
 430                                          && ack == ACK_common
 431                                          && !(commit->object.flags & COMMON)) {
 432                                                 /* We need to replay the have for this object
 433                                                  * on the next RPC request so the peer knows
 434                                                  * it is in common with us.
 435                                                  */
 436                                                 const char *hex = sha1_to_hex(result_sha1);
 437                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 438                                                 state_len = req_buf.len;
 439                                         }
 440                                         mark_common(commit, 0, 1);
 441                                         retval = 0;
 442                                         in_vain = 0;
 443                                         got_continue = 1;
 444                                         if (ack == ACK_ready) {
 445                                                 rev_list = NULL;
 446                                                 got_ready = 1;
 447                                         }
 448                                         break;
 449                                         }
 450                                 }
 451                         } while (ack);
 452                         flushes--;
 453                         if (got_continue && MAX_IN_VAIN < in_vain) {
 454                                 if (args.verbose)
 455                                         fprintf(stderr, "giving up\n");
 456                                 break; /* give up */
 457                         }
 458                 }
 459         }
 460 done:
 461         if (!got_ready || !no_done) {
 462                 packet_buf_write(&req_buf, "done\n");
 463                 send_request(fd[1], &req_buf);
 464         }
 465         if (args.verbose)
 466                 fprintf(stderr, "done\n");
 467         if (retval != 0) {
 468                 multi_ack = 0;
 469                 flushes++;
 470         }
 471         strbuf_release(&req_buf);
 472
 473         consume_shallow_list(fd[0]);
 474         while (flushes || multi_ack) {
 475                 int ack = get_ack(fd[0], result_sha1);
 476                 if (ack) {
 477                         if (args.verbose)
 478                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 479                                         sha1_to_hex(result_sha1));
 480                         if (ack == ACK)
 481                                 return 0;
 482                         multi_ack = 1;
 483                         continue;
 484                 }
 485                 flushes--;
 486         }
 487         /* it is no error to fetch into a completely empty repo */
 488         return count ? retval : 0;
 489 }
 490
 491 static struct commit_list *complete;
 492
 493 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 494 {
 495         struct object *o = parse_object(sha1);
 496
 497         while (o && o->type == OBJ_TAG) {
 498                 struct tag *t = (struct tag *) o;
 499                 if (!t->tagged)
 500                         break; /* broken repository */
 501                 o->flags |= COMPLETE;
 502                 o = parse_object(t->tagged->sha1);
 503         }
 504         if (o && o->type == OBJ_COMMIT) {
 505                 struct commit *commit = (struct commit *)o;
 506                 if (!(commit->object.flags & COMPLETE)) {
 507                         commit->object.flags |= COMPLETE;
 508                         commit_list_insert_by_date(commit, &complete);
 509                 }
 510         }
 511         return 0;
 512 }
 513
 514 static void mark_recent_complete_commits(unsigned long cutoff)
 515 {
 516         while (complete && cutoff <= complete->item->date) {
 517                 if (args.verbose)
 518                         fprintf(stderr, "Marking %s as complete\n",
 519                                 sha1_to_hex(complete->item->object.sha1));
 520                 pop_most_recent_commit(&complete, COMPLETE);
 521         }
 522 }
 523
 524 static void filter_refs(struct ref **refs, int nr_match, char **match)
 525 {
 526         struct ref **return_refs;
 527         struct ref *newlist = NULL;
 528         struct ref **newtail = &newlist;
 529         struct ref *ref, *next;
 530         struct ref *fastarray[32];
 531         int match_pos;
 532
 533         if (nr_match && !args.fetch_all) {
 534                 if (ARRAY_SIZE(fastarray) < nr_match)
 535                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 536                 else {
 537                         return_refs = fastarray;
 538                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 539                 }
 540         }
 541         else
 542                 return_refs = NULL;
 543
 544         match_pos = 0;
 545         for (ref = *refs; ref; ref = next) {
 546                 next = ref->next;
 547                 if (!memcmp(ref->name, "refs/", 5) &&
 548                     check_refname_format(ref->name + 5, 0))
 549                         ; /* trash */
 550                 else if (args.fetch_all &&
 551                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 552                         *newtail = ref;
 553                         ref->next = NULL;
 554                         newtail = &ref->next;
 555                         continue;
 556                 }
 557                 else {
 558                         int cmp = -1;
 559                         while (match_pos < nr_match) {
 560                                 cmp = strcmp(ref->name, match[match_pos]);
 561                                 if (cmp < 0) /* definitely do not have it */
 562                                         break;
 563                                 else if (cmp == 0) { /* definitely have it */
 564                                         match[match_pos][0] = '\0';
 565                                         return_refs[match_pos] = ref;
 566                                         break;
 567                                 }
 568                                 else /* might have it; keep looking */
 569                                         match_pos++;
 570                         }
 571                         if (!cmp)
 572                                 continue; /* we will link it later */
 573                 }
 574                 free(ref);
 575         }
 576
 577         if (!args.fetch_all) {
 578                 int i;
 579                 for (i = 0; i < nr_match; i++) {
 580                         ref = return_refs[i];
 581                         if (ref) {
 582                                 *newtail = ref;
 583                                 ref->next = NULL;
 584                                 newtail = &ref->next;
 585                         }
 586                 }
 587                 if (return_refs != fastarray)
 588                         free(return_refs);
 589         }
 590         *refs = newlist;
 591 }
 592
 593 static void mark_alternate_complete(const struct ref *ref, void *unused)
 594 {
 595         mark_complete(NULL, ref->old_sha1, 0, NULL);
 596 }
 597
 598 static int everything_local(struct ref **refs, int nr_match, char **match)
 599 {
 600         struct ref *ref;
 601         int retval;
 602         unsigned long cutoff = 0;
 603
 604         save_commit_buffer = 0;
 605
 606         for (ref = *refs; ref; ref = ref->next) {
 607                 struct object *o;
 608
 609                 o = parse_object(ref->old_sha1);
 610                 if (!o)
 611                         continue;
 612
 613                 /* We already have it -- which may mean that we were
 614                  * in sync with the other side at some time after
 615                  * that (it is OK if we guess wrong here).
 616                  */
 617                 if (o->type == OBJ_COMMIT) {
 618                         struct commit *commit = (struct commit *)o;
 619                         if (!cutoff || cutoff < commit->date)
 620                                 cutoff = commit->date;
 621                 }
 622         }
 623
 624         if (!args.depth) {
 625                 for_each_ref(mark_complete, NULL);
 626                 for_each_alternate_ref(mark_alternate_complete, NULL);
 627                 if (cutoff)
 628                         mark_recent_complete_commits(cutoff);
 629         }
 630
 631         /*
 632          * Mark all complete remote refs as common refs.
 633          * Don't mark them common yet; the server has to be told so first.
 634          */
 635         for (ref = *refs; ref; ref = ref->next) {
 636                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 637                                              NULL, 0);
 638
 639                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 640                         continue;
 641
 642                 if (!(o->flags & SEEN)) {
 643                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 644
 645                         mark_common((struct commit *)o, 1, 1);
 646                 }
 647         }
 648
 649         filter_refs(refs, nr_match, match);
 650
 651         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 652                 const unsigned char *remote = ref->old_sha1;
 653                 unsigned char local[20];
 654                 struct object *o;
 655
 656                 o = lookup_object(remote);
 657                 if (!o || !(o->flags & COMPLETE)) {
 658                         retval = 0;
 659                         if (!args.verbose)
 660                                 continue;
 661                         fprintf(stderr,
 662                                 "want %s (%s)\n", sha1_to_hex(remote),
 663                                 ref->name);
 664                         continue;
 665                 }
 666
 667                 hashcpy(ref->new_sha1, local);
 668                 if (!args.verbose)
 669                         continue;
 670                 fprintf(stderr,
 671                         "already have %s (%s)\n", sha1_to_hex(remote),
 672                         ref->name);
 673         }
 674         return retval;
 675 }
 676
 677 static int sideband_demux(int in, int out, void *data)
 678 {
 679         int *xd = data;
 680
 681         int ret = recv_sideband("fetch-pack", xd[0], out);
 682         close(out);
 683         return ret;
 684 }
 685
 686 static int get_pack(int xd[2], char **pack_lockfile)
 687 {
 688         struct async demux;
 689         const char *argv[20];
 690         char keep_arg[256];
 691         char hdr_arg[256];
 692         const char **av;
 693         int do_keep = args.keep_pack;
 694         struct child_process cmd;
 695
 696         memset(&demux, 0, sizeof(demux));
 697         if (use_sideband) {
 698                 /* xd[] is talking with upload-pack; subprocess reads from
 699                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 700                  * through demux->out.
 701                  */
 702                 demux.proc = sideband_demux;
 703                 demux.data = xd;
 704                 demux.out = -1;
 705                 if (start_async(&demux))
 706                         die("fetch-pack: unable to fork off sideband"
 707                             " demultiplexer");
 708         }
 709         else
 710                 demux.out = xd[0];
 711
 712         memset(&cmd, 0, sizeof(cmd));
 713         cmd.argv = argv;
 714         av = argv;
 715         *hdr_arg = 0;
 716         if (!args.keep_pack && unpack_limit) {
 717                 struct pack_header header;
 718
 719                 if (read_pack_header(demux.out, &header))
 720                         die("protocol error: bad pack header");
 721                 snprintf(hdr_arg, sizeof(hdr_arg),
 722                          "--pack_header=%"PRIu32",%"PRIu32,
 723                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 724                 if (ntohl(header.hdr_entries) < unpack_limit)
 725                         do_keep = 0;
 726                 else
 727                         do_keep = 1;
 728         }
 729
 730         if (do_keep) {
 731                 if (pack_lockfile)
 732                         cmd.out = -1;
 733                 *av++ = "index-pack";
 734                 *av++ = "--stdin";
 735                 if (!args.quiet && !args.no_progress)
 736                         *av++ = "-v";
 737                 if (args.use_thin_pack)
 738                         *av++ = "--fix-thin";
 739                 if (args.lock_pack || unpack_limit) {
 740                         int s = sprintf(keep_arg,
 741                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 742                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 743                                 strcpy(keep_arg + s, "localhost");
 744                         *av++ = keep_arg;
 745                 }
 746         }
 747         else {
 748                 *av++ = "unpack-objects";
 749                 if (args.quiet || args.no_progress)
 750                         *av++ = "-q";
 751         }
 752         if (*hdr_arg)
 753                 *av++ = hdr_arg;
 754         if (fetch_fsck_objects >= 0
 755             ? fetch_fsck_objects
 756             : transfer_fsck_objects >= 0
 757             ? transfer_fsck_objects
 758             : 0)
 759                 *av++ = "--strict";
 760         *av++ = NULL;
 761
 762         cmd.in = demux.out;
 763         cmd.git_cmd = 1;
 764         if (start_command(&cmd))
 765                 die("fetch-pack: unable to fork off %s", argv[0]);
 766         if (do_keep && pack_lockfile) {
 767                 *pack_lockfile = index_pack_lockfile(cmd.out);
 768                 close(cmd.out);
 769         }
 770
 771         if (finish_command(&cmd))
 772                 die("%s failed", argv[0]);
 773         if (use_sideband && finish_async(&demux))
 774                 die("error in sideband demultiplexer");
 775         return 0;
 776 }
 777
 778 static struct ref *do_fetch_pack(int fd[2],
 779                 const struct ref *orig_ref,
 780                 int nr_match,
 781                 char **match,
 782                 char **pack_lockfile)
 783 {
 784         struct ref *ref = copy_ref_list(orig_ref);
 785         unsigned char sha1[20];
 786
 787         sort_ref_list(&ref, ref_compare_name);
 788
 789         if (is_repository_shallow() && !server_supports("shallow"))
 790                 die("Server does not support shallow clients");
 791         if (server_supports("multi_ack_detailed")) {
 792                 if (args.verbose)
 793                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 794                 multi_ack = 2;
 795                 if (server_supports("no-done")) {
 796                         if (args.verbose)
 797                                 fprintf(stderr, "Server supports no-done\n");
 798                         if (args.stateless_rpc)
 799                                 no_done = 1;
 800                 }
 801         }
 802         else if (server_supports("multi_ack")) {
 803                 if (args.verbose)
 804                         fprintf(stderr, "Server supports multi_ack\n");
 805                 multi_ack = 1;
 806         }
 807         if (server_supports("side-band-64k")) {
 808                 if (args.verbose)
 809                         fprintf(stderr, "Server supports side-band-64k\n");
 810                 use_sideband = 2;
 811         }
 812         else if (server_supports("side-band")) {
 813                 if (args.verbose)
 814                         fprintf(stderr, "Server supports side-band\n");
 815                 use_sideband = 1;
 816         }
 817         if (server_supports("ofs-delta")) {
 818                 if (args.verbose)
 819                         fprintf(stderr, "Server supports ofs-delta\n");
 820         } else
 821                 prefer_ofs_delta = 0;
 822         if (everything_local(&ref, nr_match, match)) {
 823                 packet_flush(fd[1]);
 824                 goto all_done;
 825         }
 826         if (find_common(fd, sha1, ref) < 0)
 827                 if (!args.keep_pack)
 828                         /* When cloning, it is not unusual to have
 829                          * no common commit.
 830                          */
 831                         warning("no common commits");
 832
 833         if (args.stateless_rpc)
 834                 packet_flush(fd[1]);
 835         if (get_pack(fd, pack_lockfile))
 836                 die("git fetch-pack: fetch failed.");
 837
 838  all_done:
 839         return ref;
 840 }
 841
 842 static int remove_duplicates(int nr_heads, char **heads)
 843 {
 844         int src, dst;
 845
 846         if (!nr_heads)
 847                 return 0;
 848
 849         for (src = dst = 1; src < nr_heads; src++)
 850                 if (strcmp(heads[src], heads[dst-1]))
 851                         heads[dst++] = heads[src];
 852         return dst;
 853 }
 854
 855 static int fetch_pack_config(const char *var, const char *value, void *cb)
 856 {
 857         if (strcmp(var, "fetch.unpacklimit") == 0) {
 858                 fetch_unpack_limit = git_config_int(var, value);
 859                 return 0;
 860         }
 861
 862         if (strcmp(var, "transfer.unpacklimit") == 0) {
 863                 transfer_unpack_limit = git_config_int(var, value);
 864                 return 0;
 865         }
 866
 867         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 868                 prefer_ofs_delta = git_config_bool(var, value);
 869                 return 0;
 870         }
 871
 872         if (!strcmp(var, "fetch.fsckobjects")) {
 873                 fetch_fsck_objects = git_config_bool(var, value);
 874                 return 0;
 875         }
 876
 877         if (!strcmp(var, "transfer.fsckobjects")) {
 878                 transfer_fsck_objects = git_config_bool(var, value);
 879                 return 0;
 880         }
 881
 882         return git_default_config(var, value, cb);
 883 }
 884
 885 static struct lock_file lock;
 886
 887 static void fetch_pack_setup(void)
 888 {
 889         static int did_setup;
 890         if (did_setup)
 891                 return;
 892         git_config(fetch_pack_config, NULL);
 893         if (0 <= transfer_unpack_limit)
 894                 unpack_limit = transfer_unpack_limit;
 895         else if (0 <= fetch_unpack_limit)
 896                 unpack_limit = fetch_unpack_limit;
 897         did_setup = 1;
 898 }
 899
 900 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 901 {
 902         int i, ret;
 903         struct ref *ref = NULL;
 904         const char *dest = NULL;
 905         int alloc_heads = 0, nr_heads = 0;
 906         char **heads = NULL;
 907         int fd[2];
 908         char *pack_lockfile = NULL;
 909         char **pack_lockfile_ptr = NULL;
 910         struct child_process *conn;
 911
 912         packet_trace_identity("fetch-pack");
 913
 914         for (i = 1; i < argc && *argv[i] == '-'; i++) {
 915                 const char *arg = argv[i];
 916
 917                 if (!prefixcmp(arg, "--upload-pack=")) {
 918                         args.uploadpack = arg + 14;
 919                         continue;
 920                 }
 921                 if (!prefixcmp(arg, "--exec=")) {
 922                         args.uploadpack = arg + 7;
 923                         continue;
 924                 }
 925                 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 926                         args.quiet = 1;
 927                         continue;
 928                 }
 929                 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 930                         args.lock_pack = args.keep_pack;
 931                         args.keep_pack = 1;
 932                         continue;
 933                 }
 934                 if (!strcmp("--thin", arg)) {
 935                         args.use_thin_pack = 1;
 936                         continue;
 937                 }
 938                 if (!strcmp("--include-tag", arg)) {
 939                         args.include_tag = 1;
 940                         continue;
 941                 }
 942                 if (!strcmp("--all", arg)) {
 943                         args.fetch_all = 1;
 944                         continue;
 945                 }
 946                 if (!strcmp("--stdin", arg)) {
 947                         args.stdin_refs = 1;
 948                         continue;
 949                 }
 950                 if (!strcmp("-v", arg)) {
 951                         args.verbose = 1;
 952                         continue;
 953                 }
 954                 if (!prefixcmp(arg, "--depth=")) {
 955                         args.depth = strtol(arg + 8, NULL, 0);
 956                         continue;
 957                 }
 958                 if (!strcmp("--no-progress", arg)) {
 959                         args.no_progress = 1;
 960                         continue;
 961                 }
 962                 if (!strcmp("--stateless-rpc", arg)) {
 963                         args.stateless_rpc = 1;
 964                         continue;
 965                 }
 966                 if (!strcmp("--lock-pack", arg)) {
 967                         args.lock_pack = 1;
 968                         pack_lockfile_ptr = &pack_lockfile;
 969                         continue;
 970                 }
 971                 usage(fetch_pack_usage);
 972         }
 973
 974         if (i < argc)
 975                 dest = argv[i++];
 976         else
 977                 usage(fetch_pack_usage);
 978
 979         /*
 980          * Copy refs from cmdline to growable list, then append any
 981          * refs from the standard input:
 982          */
 983         ALLOC_GROW(heads, argc - i, alloc_heads);
 984         for (; i < argc; i++)
 985                 heads[nr_heads++] = xstrdup(argv[i]);
 986         if (args.stdin_refs) {
 987                 if (args.stateless_rpc) {
 988                         /* in stateless RPC mode we use pkt-line to read
 989                          * from stdin, until we get a flush packet
 990                          */
 991                         static char line[1000];
 992                         for (;;) {
 993                                 int n = packet_read_line(0, line, sizeof(line));
 994                                 if (!n)
 995                                         break;
 996                                 if (line[n-1] == '\n')
 997                                         n--;
 998                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
 999                                 heads[nr_heads++] = xmemdupz(line, n);
1000                         }
1001                 }
1002                 else {
1003                         /* read from stdin one ref per line, until EOF */
1004                         struct strbuf line = STRBUF_INIT;
1005                         while (strbuf_getline(&line, stdin, '\n') != EOF) {
1006                                 ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
1007                                 heads[nr_heads++] = strbuf_detach(&line, NULL);
1008                         }
1009                         strbuf_release(&line);
1010                 }
1011         }
1012
1013         if (args.stateless_rpc) {
1014                 conn = NULL;
1015                 fd[0] = 0;
1016                 fd[1] = 1;
1017         } else {
1018                 conn = git_connect(fd, dest, args.uploadpack,
1019                                    args.verbose ? CONNECT_VERBOSE : 0);
1020         }
1021
1022         get_remote_heads(fd[0], &ref, 0, NULL);
1023
1024         ref = fetch_pack(&args, fd, conn, ref, dest,
1025                 nr_heads, heads, pack_lockfile_ptr);
1026         if (pack_lockfile) {
1027                 printf("lock %s\n", pack_lockfile);
1028                 fflush(stdout);
1029         }
1030         close(fd[0]);
1031         close(fd[1]);
1032         if (finish_connect(conn))
1033                 ref = NULL;
1034         ret = !ref;
1035
1036         if (!ret && nr_heads) {
1037                 /* If the heads to pull were given, we should have
1038                  * consumed all of them by matching the remote.
1039                  * Otherwise, 'git fetch remote no-such-ref' would
1040                  * silently succeed without issuing an error.
1041                  */
1042                 for (i = 0; i < nr_heads; i++)
1043                         if (heads[i] && heads[i][0]) {
1044                                 error("no such remote ref %s", heads[i]);
1045                                 ret = 1;
1046                         }
1047         }
1048         while (ref) {
1049                 printf("%s %s\n",
1050                        sha1_to_hex(ref->old_sha1), ref->name);
1051                 ref = ref->next;
1052         }
1053
1054         return ret;
1055 }
1056
1057 static int compare_heads(const void *a, const void *b)
1058 {
1059         return strcmp(*(const char **)a, *(const char **)b);
1060 }
1061
1062 struct ref *fetch_pack(struct fetch_pack_args *my_args,
1063                        int fd[], struct child_process *conn,
1064                        const struct ref *ref,
1065                 const char *dest,
1066                 int nr_heads,
1067                 char **heads,
1068                 char **pack_lockfile)
1069 {
1070         struct stat st;
1071         struct ref *ref_cpy;
1072
1073         fetch_pack_setup();
1074         if (&args != my_args)
1075                 memcpy(&args, my_args, sizeof(args));
1076         if (args.depth > 0) {
1077                 if (stat(git_path("shallow"), &st))
1078                         st.st_mtime = 0;
1079         }
1080
1081         if (heads && nr_heads) {
1082                 qsort(heads, nr_heads, sizeof(*heads), compare_heads);
1083                 nr_heads = remove_duplicates(nr_heads, heads);
1084         }
1085
1086         if (!ref) {
1087                 packet_flush(fd[1]);
1088                 die("no matching remote head");
1089         }
1090         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1091
1092         if (args.depth > 0) {
1093                 struct cache_time mtime;
1094                 struct strbuf sb = STRBUF_INIT;
1095                 char *shallow = git_path("shallow");
1096                 int fd;
1097
1098                 mtime.sec = st.st_mtime;
1099                 mtime.nsec = ST_MTIME_NSEC(st);
1100                 if (stat(shallow, &st)) {
1101                         if (mtime.sec)
1102                                 die("shallow file was removed during fetch");
1103                 } else if (st.st_mtime != mtime.sec
1104 #ifdef USE_NSEC
1105                                 || ST_MTIME_NSEC(st) != mtime.nsec
1106 #endif
1107                           )
1108                         die("shallow file was changed during fetch");
1109
1110                 fd = hold_lock_file_for_update(&lock, shallow,
1111                                                LOCK_DIE_ON_ERROR);
1112                 if (!write_shallow_commits(&sb, 0)
1113                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1114                         unlink_or_warn(shallow);
1115                         rollback_lock_file(&lock);
1116                 } else {
1117                         commit_lock_file(&lock);
1118                 }
1119                 strbuf_release(&sb);
1120         }
1121
1122         reprepare_packed_git();
1123         return ref_cpy;
1124 }