misc/emerge-delta-webrsync

   1 #!/bin/bash
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # Author: Brian Harring <ferringb@gentoo.org>, karltk@gentoo.org originally.
   5 # Rewritten from the old, Perl-based emerge-webrsync script
   6
   7 #
   8 # gpg key import
   9 # KEY_ID=0x96D8BF6D
  10 # gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
  11 # gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
  12 #
  13
  14 argv0=$0
  15
  16 # Only echo if not in verbose mode
  17 nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
  18 # warning echos
  19 wecho() { echo "${argv0##*/}: warning: $*" 1>&2 ; }
  20 # error echos
  21 eecho() { echo "${argv0##*/}: error: $*" 1>&2 ; }
  22
  23
  24 #-------------------
  25 #initialization
  26 #------------------
  27
  28 # Use portageq from the same directory/prefix as the current script, so
  29 # that we don't have to rely on PATH including the current EPREFIX.
  30 scriptpath=${BASH_SOURCE[0]}
  31 if [ -x "${scriptpath%/*}/portageq" ]; then
  32         portageq=${scriptpath%/*}/portageq
  33 elif type -P portageq > /dev/null ; then
  34         portageq=portageq
  35 else
  36         eecho "could not find 'portageq'; aborting"
  37         exit 1
  38 fi
  39 eval "$("${portageq}" envvar -v DISTDIR EPREFIX FEATURES \
  40         FETCHCOMMAND GENTOO_MIRRORS \
  41         PORTAGE_BIN_PATH PORTAGE_CONFIGROOT PORTAGE_GPG_DIR \
  42         PORTAGE_NICENESS PORTAGE_REPOSITORIES PORTAGE_RSYNC_EXTRA_OPTS \
  43         PORTAGE_RSYNC_OPTS PORTAGE_TMPDIR \
  44         USERLAND http_proxy ftp_proxy)"
  45 export http_proxy ftp_proxy
  46
  47 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit
  48
  49 repo_name=gentoo
  50 repo_location=$(__repo_key "${repo_name}" location)
  51 if [[ -z ${repo_location} ]]; then
  52         eecho "Repository '${repo_name}' not found"
  53         exit 1
  54 fi
  55
  56 if [ -z "$NICENESS_PULLED" ]; then
  57         if [ -n "${PORTAGE_NICENESS}" ]; then
  58                 export NICENESS_PULLED=asdf
  59                 exec nice -n "${PORTAGE_NICENESS}" "$0" "$@"
  60                 echo "failed setting PORTAGE_NICENESS to '$PORTAGE_NICENESS', disabling"
  61         fi
  62 fi
  63
  64 STATE_DIR="${EPREFIX}/var/delta-webrsync/"
  65
  66 # hack.  bug 92224
  67 if [ "${FETCHCOMMAND/getdelta.sh}" != "${FETCHCOMMAND}" ]; then
  68         # evil evil evil evil
  69         eval "$(grep "^FETCHCOMMAND=" "${EPREFIX}/usr/share/portage/config/make.globals")"
  70 fi
  71
  72 unset f
  73 unset IFS
  74
  75 do_verbose=0
  76 MUST_SYNC='1'
  77 unset PUKE_HELP
  78 for x in $*; do
  79         case "${x}" in
  80                 -q|--quiet)
  81                         PORTAGE_QUIET=1
  82                         continue
  83                         ;;
  84         esac
  85         if [[ $x == "-u" ]]; then
  86                 MUST_SYNC=''
  87         elif [[ $x == "-k" ]]; then
  88                 KEEP_OLDIES='asdf'
  89         elif [[ $x == "-h" ]]; then
  90                 PUKE_HELP=1
  91         elif [[ $x == "-v" ]]; then
  92                 do_verbose=1
  93         else
  94                 PUKE_HELP=1
  95                 echo "$x isn't a valid arg.  bailing."
  96         fi
  97         if [[ -n $PUKE_HELP ]]; then
  98                 echo "-u for upgrade; sync only if new snapshots are found"
  99                 echo "-k for keep; keep old tree snapshots around"
 100                 exit -1
 101         fi
 102 done
 103
 104 if [[ ! -d $STATE_DIR ]]; then
 105         echo "$STATE_DIR doesn't exist.  don't have the ability to compensate for compressor differences without it!"
 106         exit -2
 107 fi
 108
 109 if has webrsync-gpg ${FEATURES} ; then
 110         WEBSYNC_VERIFY_SIGNATURE=1
 111 else
 112         WEBSYNC_VERIFY_SIGNATURE=0
 113 fi
 114 if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
 115         eecho "please set PORTAGE_GPG_DIR in make.conf"
 116         exit 1
 117 fi
 118
 119 [[ -d ${repo_location} ]] || mkdir -p "${repo_location}"
 120 if [[ ! -w ${repo_location} ]] ; then
 121         eecho "Repository '${repo_name}' is not writable: ${repo_location}"
 122         exit 1
 123 fi
 124
 125 [[ -d ${DISTDIR} ]] || mkdir -p "${DISTDIR}"
 126 if [[ ! -w ${DISTDIR} ]] ; then
 127         eecho "DISTDIR is not writable: ${DISTDIR}"
 128         exit 1
 129 fi
 130
 131 [[ -d ${PORTAGE_TMPDIR}/portage ]] || mkdir -p "${PORTAGE_TMPDIR}/portage"
 132 TMPDIR=$(mktemp -d "${PORTAGE_TMPDIR}/portage/delta-webrsync-XXXXXX")
 133 if [[ ! -w ${TMPDIR} ]] ; then
 134         eecho "TMPDIR is not writable: ${TMPDIR}"
 135         exit 1
 136 fi
 137
 138 cd "$DISTDIR"
 139
 140 found=0
 141
 142 if type -p md5sum > /dev/null; then
 143         md5_com='md5sum -c "${MD5_LOC}" &> /dev/null'
 144 elif type -p md5 > /dev/null; then
 145         md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \  -f 1 ${MD5_LOC})" ]'
 146 else
 147         echo "warning, unable to do md5 verification of the snapshot!"
 148         echo "no suitable md5/md5sum binary was found!"
 149         md5_com='true'
 150 fi
 151
 152 #---------------
 153 #funcs
 154 #---------------
 155
 156 cleanse_state_dir() {
 157         [[ ${STATE_DIR:-/} != '/' ]] && rm -f "${STATE_DIR}"/* &> /dev/null
 158 }
 159
 160 do_tar() {
 161         local file=$1; shift
 162         local decompressor
 163         case ${file} in
 164                 *.xz)   decompressor="xzcat" ;;
 165                 *.bz2)  decompressor="bzcat" ;;
 166                 *.gz)   decompressor="zcat"  ;;
 167                 *)      decompressor="cat"   ;;
 168         esac
 169         ${decompressor} "${file}" | tar "$@"
 170         _pipestatus=${PIPESTATUS[*]}
 171         [[ ${_pipestatus// /} -eq 0 ]]
 172 }
 173
 174 get_utc_date_in_seconds() {
 175         date -u +"%s"
 176 }
 177
 178 get_date_part() {
 179         local utc_time_in_secs="$1"
 180         local part="$2"
 181
 182         if      [[ ${USERLAND} == BSD ]] ; then
 183                 date -r ${utc_time_in_secs} -u +"${part}"
 184         else
 185                 date -d @${utc_time_in_secs} -u +"${part}"
 186         fi
 187 }
 188
 189 get_utc_second_from_string() {
 190         local s="$1"
 191         if [[ ${USERLAND} == BSD ]] ; then
 192                 # Specify zeros for the least significant digits, or else those
 193                 # digits are inherited from the current system clock time.
 194                 date -juf "%Y%m%d%H%M.%S" "${s}0000.00" +"%s"
 195         else
 196                 date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s"
 197         fi
 198 }
 199
 200 get_portage_timestamp() {
 201         local portage_current_timestamp=0
 202
 203         if [ -f "${repo_location}/metadata/timestamp.x" ]; then
 204                 portage_current_timestamp=$(cut -f 1 -d " " "${repo_location}/metadata/timestamp.x" )
 205         fi
 206
 207         echo "${portage_current_timestamp}"
 208 }
 209
 210 increment_date() {
 211         local s="$1" inc="$2"
 212         if [[ ${USERLAND} == BSD ]] ; then
 213                 # Specify zeros for the least significant digits, or else those
 214                 # digits are inherited from the current system clock time.
 215                 date -v${inc}d -juf "%Y%m%d%H%M.%S" "${s}0000.00" +"%Y%m%d"
 216         else
 217                 date -d "${s:0:4}-${s:4:2}-${s:6:2} ${inc} day" -u +"%Y%m%d"
 218         fi
 219 }
 220
 221
 222 fetch_file() {
 223         local URI="$1"
 224         local FILE="$2"
 225         local opts
 226
 227         if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
 228                 opts="--continue $(nvecho -q)"
 229         elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
 230                 opts="--continue-at - $(nvecho -s -f)"
 231         else
 232                 rm -f "${DISTDIR}/${FILE}"
 233         fi
 234
 235         __vecho "Fetching file ${FILE} ..."
 236         # already set DISTDIR=
 237         eval "${FETCHCOMMAND} ${opts}"
 238         if [[ $? -eq 0 && -s ${DISTDIR}/${FILE} ]] ; then
 239                 return 0
 240         else
 241                 rm -f "${DISTDIR}/${FILE}"
 242                 return 1
 243         fi
 244 }
 245
 246 check_file_digest() {
 247         local digest="$1"
 248         local file="$2"
 249         local r=1
 250
 251         __vecho "Checking digest ..."
 252
 253         if type -P md5sum > /dev/null; then
 254                 local md5sum_output=$(md5sum "${file}")
 255                 local digest_content=$(< "${digest}")
 256                 [ "${md5sum_output%%[[:space:]]*}" = "${digest_content%%[[:space:]]*}" ] && r=0
 257         elif type -P md5 > /dev/null; then
 258                 [ "$(md5 -q "${file}")" == "$(cut -d ' ' -f 1 "${digest}")" ] && r=0
 259         else
 260                 eecho "cannot check digest: no suitable md5/md5sum binaries found"
 261         fi
 262
 263         return "${r}"
 264 }
 265
 266 check_file_signature() {
 267         local signature="$1"
 268         local file="$2"
 269         local r=1
 270
 271         if [[ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]] ; then
 272
 273                 __vecho "Checking signature ..."
 274
 275                 if type -P gpg > /dev/null; then
 276                         gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
 277                 else
 278                         eecho "cannot check signature: gpg binary not found"
 279                         exit 1
 280                 fi
 281         else
 282                 r=0
 283         fi
 284
 285         return "${r}"
 286 }
 287
 288 get_snapshot_timestamp() {
 289         local file="$1"
 290
 291         do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
 292 }
 293
 294 sync_local() {
 295         local file="$1"
 296
 297         __vecho "Syncing local tree ..."
 298
 299         local ownership="portage:portage"
 300         if has usersync ${FEATURES} ; then
 301                 case "${USERLAND}" in
 302                         BSD)
 303                                 ownership=$(stat -f '%Su:%Sg' "${repo_location}")
 304                                 ;;
 305                         *)
 306                                 ownership=$(stat -c '%U:%G' "${repo_location}")
 307                                 ;;
 308                 esac
 309         fi
 310
 311         if type -P tarsync > /dev/null ; then
 312                 local chown_opts="-o ${ownership%:*} -g ${ownership#*:}"
 313                 chown ${ownership} "${repo_location}" > /dev/null 2>&1 || chown_opts=""
 314                 if ! tarsync $(__vecho -v) -s 1 ${chown_opts} \
 315                         -e /distfiles -e /packages -e /local "${file}" "${repo_location}"; then
 316                         eecho "tarsync failed; tarball is corrupt? (${file})"
 317                         return 1
 318                 fi
 319         else
 320                 if ! do_tar "${file}" xf - -C "${TMPDIR}" ; then
 321                         eecho "tar failed to extract the image. tarball is corrupt? (${file})"
 322                         rm -fr "${TMPDIR}"/portage
 323                         return 1
 324                 fi
 325
 326                 local rsync_opts="${PORTAGE_RSYNC_OPTS} ${PORTAGE_RSYNC_EXTRA_OPTS}"
 327                 if chown ${ownership} "${TMPDIR}"/portage > /dev/null 2>&1; then
 328                         chown -R ${ownership} "${TMPDIR}"/portage
 329                         rsync_opts+=" --owner --group"
 330                 fi
 331                 cd "${TMPDIR}"/portage
 332                 rsync ${rsync_opts} . "${repo_location%%/}"
 333                 cd "${DISTDIR}"
 334
 335                 __vecho "Cleaning up ..."
 336                 rm -fr "${TMPDIR}"
 337         fi
 338
 339         if has metadata-transfer ${FEATURES} ; then
 340                 __vecho "Updating cache ..."
 341                 "${PORTAGE_BIN_PATH}/emerge" --metadata
 342         fi
 343         local post_sync=${PORTAGE_CONFIGROOT}etc/portage/bin/post_sync
 344         [ -x "${post_sync}" ] && "${post_sync}"
 345         # --quiet suppresses output if there are no relevant news items
 346         has news ${FEATURES} && "${PORTAGE_BIN_PATH}/emerge" --check-news --quiet
 347         return 0
 348 }
 349
 350 do_snapshot() {
 351         local ignore_timestamp="$1"
 352         local date="$2"
 353
 354         local r=1
 355
 356         local base_file="portage-${date}.tar"
 357
 358         local have_files=0
 359         local mirror
 360
 361         local compressions=""
 362         type -P bzcat > /dev/null && compressions="${compressions} bz2"
 363
 364         if [[ -z ${compressions} ]] ; then
 365                 eecho "unable to locate any decompressors (xzcat or bzcat or zcat)"
 366                 exit 1
 367         fi
 368
 369         for mirror in ${GENTOO_MIRRORS} ; do
 370
 371                 mirror=${mirror%/}
 372                 __vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."
 373
 374                 for compression in ${compressions} ; do
 375                         local file="portage-${date}.tar.${compression}"
 376                         local digest="${file}.md5sum"
 377                         local signature="${file}.gpgsig"
 378
 379                         if [ -s "${DISTDIR}/${file}" -a -s "${DISTDIR}/${digest}" -a -s "${DISTDIR}/${signature}" ] ; then
 380                                 check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
 381                                 check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
 382                                 have_files=1
 383                         fi
 384
 385                         if [ ${have_files} -eq 0 ] ; then
 386                                 fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
 387                                 fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
 388                                 fetch_file "${mirror}/snapshots/${file}" "${file}" && \
 389                                 check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
 390                                 check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
 391                                 have_files=1
 392                         fi
 393
 394                         #
 395                         # If timestamp is invalid
 396                         # we want to try and retrieve
 397                         # from a different mirror
 398                         #
 399                         if [ ${have_files} -eq 1 ]; then
 400
 401                                 __vecho "Getting snapshot timestamp ..."
 402                                 local snapshot_timestamp=$(get_snapshot_timestamp "${DISTDIR}/${file}")
 403
 404                                 if [ ${ignore_timestamp} == 0 ]; then
 405                                         if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
 406                                                 wecho "portage is newer than snapshot"
 407                                                 have_files=0
 408                                         fi
 409                                 else
 410                                         local utc_seconds=$(get_utc_second_from_string "${date}")
 411
 412                                         #
 413                                         # Check that this snapshot
 414                                         # is what it claims to be ...
 415                                         #
 416                                         if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
 417                                                 [ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
 418
 419                                                 wecho "snapshot timestamp is not in acceptable period"
 420                                                 have_files=0
 421                                         fi
 422                                 fi
 423                         fi
 424
 425                         if [ ${have_files} -eq 1 ]; then
 426                                 break
 427                         else
 428                                 #
 429                                 # Remove files and use a different mirror
 430                                 #
 431                                 rm -f "${DISTDIR}/${file}" "${DISTDIR}/${digest}" "${DISTDIR}/${signature}"
 432                         fi
 433                 done
 434
 435                 [ ${have_files} -eq 1 ] && break
 436         done
 437
 438         if [ ${have_files} -eq 1 ]; then
 439                 sync_local "${DISTDIR}/${file}" && r=0
 440         else
 441                 __vecho "${date} snapshot was not found"
 442         fi
 443
 444         return "${r}"
 445 }
 446
 447 do_latest_snapshot() {
 448         local attempts=0
 449         local r=1
 450
 451         __vecho "Fetching most recent snapshot ..."
 452
 453         # The snapshot for a given day is generated at 00:45 UTC on the following
 454         # day, so the current day's snapshot (going by UTC time) hasn't been
 455         # generated yet.  Therefore, always start by looking for the previous day's
 456         # snapshot (for attempts=1, subtract 1 day from the current UTC time).
 457
 458         # Timestamps that differ by less than 2 hours
 459         # are considered to be approximately equal.
 460         local min_time_diff=$(( 2 * 60 * 60 ))
 461
 462         local existing_timestamp=$(get_portage_timestamp)
 463         local timestamp_difference
 464         local timestamp_problem
 465         local approx_snapshot_time
 466         local start_time=$(get_utc_date_in_seconds)
 467         local start_hour=$(get_date_part ${start_time} "%H")
 468
 469         # Daily snapshots are created at 00:45 and are not
 470         # available until after 01:00. Don't waste time trying
 471         # to fetch a snapshot before it's been created.
 472         if [ ${start_hour} -lt 1 ] ; then
 473                 (( start_time -= 86400 ))
 474         fi
 475         local snapshot_date=$(get_date_part ${start_time} "%Y%m%d")
 476         local snapshot_date_seconds=$(get_utc_second_from_string ${snapshot_date})
 477
 478         while (( ${attempts} <  40 )) ; do
 479                 (( attempts++ ))
 480                 (( snapshot_date_seconds -= 86400 ))
 481                 # snapshots are created at 00:45
 482                 (( approx_snapshot_time = snapshot_date_seconds + 86400 + 2700 ))
 483                 (( timestamp_difference = existing_timestamp - approx_snapshot_time ))
 484                 [ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference ))
 485                 snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d")
 486
 487                 timestamp_problem=""
 488                 if [ ${timestamp_difference} -eq 0 ]; then
 489                         timestamp_problem="is identical to"
 490                 elif [ ${timestamp_difference} -lt ${min_time_diff} ]; then
 491                         timestamp_problem="is possibly identical to"
 492                 elif [ ${approx_snapshot_time} -lt ${existing_timestamp} ] ; then
 493                         timestamp_problem="is newer than"
 494                 fi
 495
 496                 if [ -n "${timestamp_problem}" ]; then
 497                         ewarn "Latest snapshot date: ${snapshot_date}"
 498                         ewarn
 499                         ewarn "Approximate snapshot timestamp: ${approx_snapshot_time}"
 500                         ewarn "       Current local timestamp: ${existing_timestamp}"
 501                         ewarn
 502                         echo -e "The current local timestamp" \
 503                                 "${timestamp_problem} the" \
 504                                 "timestamp of the latest" \
 505                                 "snapshot. In order to force sync," \
 506                                 "use the --revert option or remove" \
 507                                 "the timestamp file located at" \
 508                                 "'${repo_location}/metadata/timestamp.x'." | fmt -w 70 | \
 509                                 while read -r line ; do
 510                                         ewarn "${line}"
 511                                 done
 512                         r=0
 513                         break
 514                 fi
 515
 516                 if do_snapshot 0 "${snapshot_date}"; then
 517                         r=0
 518                         break;
 519                 fi
 520         done
 521
 522         return "${r}"
 523 }
 524
 525 fetch_from_mirrors() {
 526         local i URI FILE MIRRORS
 527         if [[ "$#" == 3 ]]; then
 528                 MIRRORS="${3}"
 529         else
 530                 MIRRORS=$GENTOO_MIRRORS
 531         fi
 532         FILE="$2"
 533         for i in $MIRRORS ; do
 534                 URI="${i%/}/${1#/}"
 535                 fetch_file "${URI}" "${FILE}" && return 0
 536         done
 537         return 1
 538 }
 539
 540 verify_md5_file() {
 541         local FILE MD5_LOC
 542         FILE="$1"
 543         if [[ $# == 2 ]]; then
 544                 MD5_LOC="$2"
 545         else
 546                 MD5_LOC="$(pwd)/$1.md5sum"
 547         fi
 548         check_file_digest "${MD5_LOC}" "${FILE}"
 549 }
 550
 551 #--------------------
 552 #inline actual script
 553 #--------------------
 554
 555 if ! type -p patcher &> /dev/null; then
 556         echo "!!!"
 557         echo "!!! cannot find patcher, did you emerge dev-util/diffball?"
 558         echo "!!! lack of patcher == have to do full fetch"
 559         echo "!!!"
 560         sleep 10
 561         if do_latest_snapshot; then
 562                 rm -fr "${TMPDIR}"
 563                 cleanse_state_dir
 564                 exit 0
 565         fi
 566         exit 1
 567 fi
 568
 569 echo "Looking for available base versions for a delta"
 570
 571 #note we're already in distdir
 572
 573 unset base_version
 574 # portage-snapshots in reverse order.
 575 # icky.
 576 unset dfile
 577 potentials="$(ls -1 portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 ${STATE_DIR}/portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 2> /dev/null | sed -e 's:^.*/::' | sort -r)"
 578 for basef in ${potentials}; do
 579         chksum=''
 580         found="dar"
 581         if [ -e "${STATE_DIR}/${basef}.md5sum" ]; then
 582                 chksum="${STATE_DIR}/${basef}.md5sum"
 583         elif [ -e "${basef}.md5sum" ]; then
 584                 chksum="${DISTDIR}/${basef}.md5sum"
 585         else
 586                 echo "attempting to get md5sum for $basef"
 587                 if ! fetch_from_mirrors "/snapshots/${basef}.md5sum" "${basef}.md5sum"; then
 588                         echo "can't get md5 for ${basef}"
 589                         continue
 590                 fi
 591                 chksum="${basef}.md5sum"
 592         fi
 593         if [ -e "${basef}" ]; then
 594                 dfile="${DISTDIR}/${basef}"
 595         else
 596                 dfile="${STATE_DIR}/${basef}"
 597         fi
 598         if ! verify_md5_file "${dfile}" "${chksum}"; then
 599                 echo "found a stale snapshot.  cleansing"
 600                 rm -f "${dfile}" &> /dev/null
 601                 rm -f "${chksum}.md5sum" &> /dev/null
 602                 dar=""
 603         else
 604                 base_version="${basef}"
 605                 break
 606         fi
 607 done
 608
 609 #by this point, we either have a base_version, or we don't.
 610 if [[ -z ${base_version} ]]; then
 611         echo "no base found.  resorting to pulling a full version"
 612         if do_latest_snapshot; then
 613                 rm -fr "${TMPDIR}"
 614                 cleanse_state_dir
 615                 exit 0
 616         fi
 617         exit 1
 618 fi
 619
 620 #we have a md5 verified base.  now we get the patch.
 621
 622 base_date="${base_version%.tar.bz2}"
 623 base_date="${base_date#portage-}"
 624 # we now have yyyymmdd
 625
 626 patches=''
 627 echo "fetching patches"
 628 fetched='asdf'
 629 while [[ -n ${fetched} ]]; do
 630         next_day=$(increment_date ${base_date} +1)
 631         # if we can't get a *single* patch or md5, even one missing, do full.
 632         p="snapshot-${base_date}-${next_day}.patch.bz2"
 633         if [[ ! -e ${p}.md5sum ]] && ! fetch_from_mirrors "/snapshots/deltas/${p}.md5sum" "${p}.md5sum"; then
 634                 echo "failed fetching ${p}.md5sum"
 635                 fetched=''
 636                 break
 637         fi
 638         fetch="yes"
 639         if [[ -e ${p} ]]; then
 640                 if ! verify_md5_file "${p}"; then
 641                         rm -f "${p}" &> /dev/null
 642                 else
 643                         fetch=""
 644                 fi
 645         fi
 646         if [[ -n $fetch ]]; then
 647                 if ! fetch_from_mirrors "/snapshots/deltas/${p}" "${p}"; then
 648                         echo "failed fetching ${p}"
 649                         fetched=''
 650                 fi
 651         fi
 652         if [[ -z ${fetched} ]]; then
 653                 break
 654         fi
 655         if ! verify_md5_file "${p}"; then
 656                 echo "md5 failed on ${p}"
 657                 fetched=''
 658                 break
 659         fi
 660         patches="${patches} ${p}"
 661         base_date="${next_day}"
 662 done
 663 final_date=${base_date}
 664
 665 if [[ -z $patches ]]; then
 666         echo "no patches found? up to date?"
 667         if [[ -n $MUST_SYNC ]]; then
 668                 echo "syncing with existing file"
 669                 if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 &&
 670                         ! -e ${DISTDIR}/portage-${base_date}.tar.bz2.gpgsig ]] && \
 671                         ! fetch_from_mirrors "/snapshots/portage-${base_date}.tar.bz2.gpgsig" "portage-${base_date}.tar.bz2.gpgsig" ; then
 672                         eecho "Couldn't fetch portage-${base_date}.tar.bz2.gpgsig"
 673                         exit 5
 674                 fi
 675                 if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
 676                         check_file_signature "${DISTDIR}/portage-${base_date}.tar.bz2.gpgsig" "${dfile}" || exit 1
 677                 fi
 678                 sync_local "${dfile}" && rm -fr "${TMPDIR}"
 679         else
 680                 rm -fr "${TMPDIR}"
 681         fi
 682         exit $?
 683 fi
 684
 685 unset got_umd5
 686 #grab the md5 for later usage.
 687 if [[ ! -e portage-${final_date}.tar.bz2.md5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.md5sum" "portage-${final_date}.tar.bz2.md5sum"; then
 688         echo "warning... couldn't grab the md5sum for ${final_date}.  which is odd"
 689         echo "thus, bailing (sorry)"
 690         exit 5
 691 else
 692         if [[ ! -e portage-${final_date}.tar.bz2.umd5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then
 693                 if ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then
 694                         echo "couldn't grab umd5sum (uncompressed md5sum) for ${final_date}."
 695                         echo "can't compensate for bzip2 version differences iow."
 696                 else
 697                         got_umd5=1
 698                 fi
 699         else
 700                 got_umd5=1
 701         fi
 702 fi
 703
 704 if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 && ! -e portage-${final_date}.tar.bz2.gpgsig ]] && \
 705         ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.gpgsig" "portage-${final_date}.tar.bz2.gpgsig" ; then
 706         echo "warning... couldn't grab the gpgsig for ${final_date}.  which is odd"
 707         echo "thus, bailing (sorry)"
 708         exit 5
 709 fi
 710
 711 # got our patches.
 712 if ! patcher -v "${dfile}" ${patches} "${TMPDIR}/portage-${final_date}.tar"; then
 713         echo "reconstruction failed (contact the author with the error from the reconstructor please)"
 714         rm -f "${TMPDIR}/portage-${final_date}.tar"
 715         if do_latest_snapshot; then
 716                 rm -fr "${TMPDIR}"
 717                 cleanse_state_dir
 718                 exit 0
 719         fi
 720         exit 1
 721 fi
 722 verified=0
 723 if [[ -n $got_umd5 ]]; then
 724         echo "verifying uncompressed md5"
 725         if ! verify_md5_file "${TMPDIR}/portage-${final_date}.tar" "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum"; then
 726                 echo "uncompressed verification failed.  This means either you found a bug in diffball, or something odd is going on"
 727                 echo "with upstream patch generation"
 728                 echo "trying md5sum next, which probably will fail."
 729         else
 730                 verified="1"
 731         fi
 732 fi
 733
 734 unset need_last_sync
 735 if [ "$verified" == "1" ]; then
 736         need_last_sync="dar"
 737         if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
 738                 # BUG: Signature verification will fail if the local bzip2
 739                 # program does not produce output that is perfectly identical
 740                 # to the bzip2 program used to compress the signed tar file.
 741                 echo "recompressing ..."
 742                 bzip2 -vk9 "${TMPDIR}/portage-${final_date}.tar"
 743                 check_file_signature "${DISTDIR}/portage-${final_date}.tar.bz2.gpgsig" "${TMPDIR}/portage-${final_date}.tar.bz2" || exit 1
 744         else
 745                 echo "recompressing. (backgrounding)"
 746                 bzip2 -vk9 "${TMPDIR}/portage-${final_date}.tar" &
 747         fi
 748
 749         echo "beginning update to the tree"
 750         sync_local "${TMPDIR}/portage-${final_date}.tar"
 751         echo "doing final md5 stuff"
 752         wait
 753         # bzip2 is finished now.
 754         rm -f "${TMPDIR}/portage-${final_date}.tar"
 755 else
 756         echo "recompressing."
 757         bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar.bz2"
 758 fi
 759
 760 echo "verifying generated tarball"
 761
 762 if ! verify_md5_file "${TMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/portage-${final_date}.tar.bz2.md5sum"; then
 763         if [[ -z $verified ]]; then
 764                 echo "couldn't verify the generated tarball.  bug, most likely."
 765                 exit 5
 766         fi
 767         # hokay.  md5 doesn't agree with umd5. bzip2 issue in effect.
 768         echo "compressed md5 differs, but uncompressed md5 says it right.  bzip2 version incompatability in other words"
 769         echo "saving the md5"
 770         if type -p md5sum &> /dev/null; then
 771                 md5sum "${TMPDIR}/portage-${final_date}.tar.bz2" | sed -e "s:${TMPDIR}/\?::" > \
 772                         "${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum"
 773         elif type -p md5 &> /dev/null; then
 774                 echo "$(md5 -q "${TMPDIR}/portage-${final_date}.tar.bz2")  portage-${final_date}.tar.bz2" > \
 775                         "${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum"
 776         else
 777                 echo "couldn't find either md5 or md5sum.  something is screwed... (bailing, sorry)"
 778                 exit 7
 779         fi
 780         mv "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum" "${TMPDIR}/portage-${final_date}.tar.bz2" "${STATE_DIR}/"
 781         dfile="${STATE_DIR}/portage-${final_date}.tar.bz2"
 782 else
 783         dfile="${DISTDIR}/portage-${final_date}.tar.bz2"
 784         mv "${TMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/"
 785 fi
 786
 787 if [ -z "${need_last_sync}" ]; then
 788         if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
 789                 check_file_signature "${DISTDIR}/portage-${final_date}.tar.bz2.gpgsig" "${dfile}" || exit 1
 790         fi
 791         echo "beginning update to the tree"
 792         sync_local "${dfile}"
 793 fi
 794
 795 for x in ${patches} ; do
 796         rm -f "${DISTDIR}/${x}"{,.md5sum}
 797 done
 798
 799 if [[ -z $KEEP_OLDIES ]]; then
 800         echo "cleansing"
 801         for x in $potentials; do
 802                 echo "removing ${x}"
 803                 rm -f "${DISTDIR}/${x}"{,.md5sum,.umd5sum,.gpgsig} &> /dev/null
 804                 rm -f "${STATE_DIR}/${x}"{,.md5sum,.umd5sum} &> /dev/null
 805         done
 806 fi
 807 rm -rf "${TMPDIR}"
 808 echo "done."
 809