misc/emerge-delta-webrsync

   1 #!/bin/bash
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # Author: Brian Harring <ferringb@gentoo.org>, karltk@gentoo.org originally.
   5 # Rewritten from the old, Perl-based emerge-webrsync script
   6
   7 #
   8 # gpg key import
   9 # KEY_ID=0x96D8BF6D
  10 # gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
  11 # gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
  12 #
  13
  14 argv0=$0
  15
  16 # Only echo if not in verbose mode
  17 nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
  18 # warning echos
  19 wecho() { echo "${argv0##*/}: warning: $*" 1>&2 ; }
  20 # error echos
  21 eecho() { echo "${argv0##*/}: error: $*" 1>&2 ; }
  22
  23
  24 #-------------------
  25 #initialization
  26 #------------------
  27
  28 # Use portageq from the same directory/prefix as the current script, so
  29 # that we don't have to rely on PATH including the current EPREFIX.
  30 scriptpath=${BASH_SOURCE[0]}
  31 if [ -x "${scriptpath%/*}/portageq" ]; then
  32         portageq=${scriptpath%/*}/portageq
  33 elif type -P portageq > /dev/null ; then
  34         portageq=portageq
  35 else
  36         eecho "could not find 'portageq'; aborting"
  37         exit 1
  38 fi
  39 eval $("${portageq}" envvar -v DISTDIR EPREFIX FEATURES \
  40         FETCHCOMMAND GENTOO_MIRRORS \
  41         PORTAGE_BIN_PATH PORTAGE_CONFIGROOT PORTAGE_GPG_DIR \
  42         PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS \
  43         PORTAGE_RSYNC_OPTS PORTAGE_TMPDIR PORTDIR \
  44         USERLAND http_proxy ftp_proxy)
  45 export http_proxy ftp_proxy
  46
  47 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
  48
  49 if [ -z "$NICENESS_PULLED" ]; then
  50         if [ -n "${PORTAGE_NICENESS}" ]; then
  51                 export NICENESS_PULLED=asdf
  52                 exec nice -n "${PORTAGE_NICENESS}" "$0" "$@"
  53                 echo "failed setting PORTAGE_NICENESS to '$PORTAGE_NICENESS', disabling"
  54         fi
  55 fi
  56
  57 STATE_DIR="${EPREFIX}/var/delta-webrsync/"
  58
  59 # hack.  bug 92224
  60 if [ "${FETCHCOMMAND/getdelta.sh}" != "${FETCHCOMMAND}" ]; then
  61         # evil evil evil evil
  62         eval "$(grep "^FETCHCOMMAND=" "${EPREFIX}/usr/share/portage/config/make.globals")"
  63 fi
  64
  65 unset f
  66 unset IFS
  67
  68 do_verbose=0
  69 MUST_SYNC='1'
  70 unset PUKE_HELP
  71 for x in $*; do
  72         case "${x}" in
  73                 -q|--quiet)
  74                         PORTAGE_QUIET=1
  75                         continue
  76                         ;;
  77         esac
  78         if [[ $x == "-u" ]]; then
  79                 MUST_SYNC=''
  80         elif [[ $x == "-k" ]]; then
  81                 KEEP_OLDIES='asdf'
  82         elif [[ $x == "-h" ]]; then
  83                 PUKE_HELP=1
  84         elif [[ $x == "-v" ]]; then
  85                 do_verbose=1
  86         else
  87                 PUKE_HELP=1
  88                 echo "$x isn't a valid arg.  bailing."
  89         fi
  90         if [[ -n $PUKE_HELP ]]; then
  91                 echo "-u for upgrade; sync only if new snapshots are found"
  92                 echo "-k for keep; keep old tree snapshots around"
  93                 exit -1
  94         fi
  95 done
  96
  97 if [[ ! -d $STATE_DIR ]]; then
  98         echo "$STATE_DIR doesn't exist.  don't have the ability to compensate for compressor differences without it!"
  99         exit -2
 100 fi
 101
 102 if has webrsync-gpg ${FEATURES} ; then
 103         WEBSYNC_VERIFY_SIGNATURE=1
 104 else
 105         WEBSYNC_VERIFY_SIGNATURE=0
 106 fi
 107 if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
 108         eecho "please set PORTAGE_GPG_DIR in make.conf"
 109         exit 1
 110 fi
 111
 112 [[ -d ${PORTDIR} ]] || mkdir -p "${PORTDIR}"
 113 if [[ ! -w ${PORTDIR} ]] ; then
 114         eecho "PORTDIR is not writable: ${PORTDIR}"
 115         exit 1
 116 fi
 117
 118 [[ -d ${DISTDIR} ]] || mkdir -p "${DISTDIR}"
 119 if [[ ! -w ${DISTDIR} ]] ; then
 120         eecho "DISTDIR is not writable: ${DISTDIR}"
 121         exit 1
 122 fi
 123
 124 [[ -d ${PORTAGE_TMPDIR}/portage ]] || mkdir -p "${PORTAGE_TMPDIR}/portage"
 125 TMPDIR=$(mktemp -d "${PORTAGE_TMPDIR}/portage/delta-webrsync-XXXXXX")
 126 if [[ ! -w ${TMPDIR} ]] ; then
 127         eecho "TMPDIR is not writable: ${TMPDIR}"
 128         exit 1
 129 fi
 130
 131 cd "$DISTDIR"
 132
 133 found=0
 134
 135 if type -p md5sum > /dev/null; then
 136         md5_com='md5sum -c "${MD5_LOC}" &> /dev/null'
 137 elif type -p md5 > /dev/null; then
 138         md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \  -f 1 ${MD5_LOC})" ]'
 139 else
 140         echo "warning, unable to do md5 verification of the snapshot!"
 141         echo "no suitable md5/md5sum binary was found!"
 142         md5_com='true'
 143 fi
 144
 145 #---------------
 146 #funcs
 147 #---------------
 148
 149 cleanse_state_dir() {
 150         [[ ${STATE_DIR:-/} != '/' ]] && rm -f "${STATE_DIR}"/* &> /dev/null
 151 }
 152
 153 do_tar() {
 154         local file=$1; shift
 155         local decompressor
 156         case ${file} in
 157                 *.xz)   decompressor="xzcat" ;;
 158                 *.bz2)  decompressor="bzcat" ;;
 159                 *.gz)   decompressor="zcat"  ;;
 160                 *)      decompressor="cat"   ;;
 161         esac
 162         ${decompressor} "${file}" | tar "$@"
 163         _pipestatus=${PIPESTATUS[*]}
 164         [[ ${_pipestatus// /} -eq 0 ]]
 165 }
 166
 167 get_utc_date_in_seconds() {
 168         date -u +"%s"
 169 }
 170
 171 get_date_part() {
 172         local utc_time_in_secs="$1"
 173         local part="$2"
 174
 175         if      [[ ${USERLAND} == BSD ]] ; then
 176                 date -r ${utc_time_in_secs} -u +"${part}"
 177         else
 178                 date -d @${utc_time_in_secs} -u +"${part}"
 179         fi
 180 }
 181
 182 get_utc_second_from_string() {
 183         local s="$1"
 184         if [[ ${USERLAND} == BSD ]] ; then
 185                 # Specify zeros for the least significant digits, or else those
 186                 # digits are inherited from the current system clock time.
 187                 date -juf "%Y%m%d%H%M.%S" "${s}0000.00" +"%s"
 188         else
 189                 date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s"
 190         fi
 191 }
 192
 193 get_portage_timestamp() {
 194         local portage_current_timestamp=0
 195
 196         if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then
 197                 portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" )
 198         fi
 199
 200         echo "${portage_current_timestamp}"
 201 }
 202
 203 increment_date() {
 204         local s="$1" inc="$2"
 205         if [[ ${USERLAND} == BSD ]] ; then
 206                 # Specify zeros for the least significant digits, or else those
 207                 # digits are inherited from the current system clock time.
 208                 date -v${inc}d -juf "%Y%m%d%H%M.%S" "${s}0000.00" +"%Y%m%d"
 209         else
 210                 date -d "${s:0:4}-${s:4:2}-${s:6:2} ${inc} day" -u +"%Y%m%d"
 211         fi
 212 }
 213
 214
 215 fetch_file() {
 216         local URI="$1"
 217         local FILE="$2"
 218         local opts
 219
 220         if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
 221                 opts="--continue $(nvecho -q)"
 222         elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
 223                 opts="--continue-at - $(nvecho -s -f)"
 224         else
 225                 rm -f "${FILE}"
 226         fi
 227
 228         __vecho "Fetching file ${FILE} ..."
 229         # already set DISTDIR=
 230         eval "${FETCHCOMMAND} ${opts}"
 231         [ -s "${DISTDIR}/${FILE}" ]
 232 }
 233
 234 check_file_digest() {
 235         local digest="$1"
 236         local file="$2"
 237         local r=1
 238
 239         __vecho "Checking digest ..."
 240
 241         if type -P md5sum > /dev/null; then
 242                 local md5sum_output=$(md5sum "${file}")
 243                 local digest_content=$(< "${digest}")
 244                 [ "${md5sum_output%%[[:space:]]*}" = "${digest_content%%[[:space:]]*}" ] && r=0
 245         elif type -P md5 > /dev/null; then
 246                 [ "$(md5 -q "${file}")" == "$(cut -d ' ' -f 1 "${digest}")" ] && r=0
 247         else
 248                 eecho "cannot check digest: no suitable md5/md5sum binaries found"
 249         fi
 250
 251         return "${r}"
 252 }
 253
 254 check_file_signature() {
 255         local signature="$1"
 256         local file="$2"
 257         local r=1
 258
 259         if [[ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]] ; then
 260
 261                 __vecho "Checking signature ..."
 262
 263                 if type -P gpg > /dev/null; then
 264                         gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
 265                 else
 266                         eecho "cannot check signature: gpg binary not found"
 267                         exit 1
 268                 fi
 269         else
 270                 r=0
 271         fi
 272
 273         return "${r}"
 274 }
 275
 276 get_snapshot_timestamp() {
 277         local file="$1"
 278
 279         do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
 280 }
 281
 282 sync_local() {
 283         local file="$1"
 284
 285         __vecho "Syncing local tree ..."
 286
 287         local ownership="portage:portage"
 288         if has usersync ${FEATURES} ; then
 289                 case "${USERLAND}" in
 290                         BSD)
 291                                 ownership=$(stat -f '%Su:%Sg' "${PORTDIR}")
 292                                 ;;
 293                         *)
 294                                 ownership=$(stat -c '%U:%G' "${PORTDIR}")
 295                                 ;;
 296                 esac
 297         fi
 298
 299         if type -P tarsync > /dev/null ; then
 300                 local chown_opts="-o ${ownership%:*} -g ${ownership#*:}"
 301                 chown ${ownership} "${PORTDIR}" > /dev/null 2>&1 || chown_opts=""
 302                 if ! tarsync $(__vecho -v) -s 1 ${chown_opts} \
 303                         -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
 304                         eecho "tarsync failed; tarball is corrupt? (${file})"
 305                         return 1
 306                 fi
 307         else
 308                 if ! do_tar "${file}" xf - -C "${TMPDIR}" ; then
 309                         eecho "tar failed to extract the image. tarball is corrupt? (${file})"
 310                         rm -fr "${TMPDIR}"/portage
 311                         return 1
 312                 fi
 313
 314                 local rsync_opts="${PORTAGE_RSYNC_OPTS} ${PORTAGE_RSYNC_EXTRA_OPTS}"
 315                 if chown ${ownership} "${TMPDIR}"/portage > /dev/null 2>&1; then
 316                         chown -R ${ownership} "${TMPDIR}"/portage
 317                         rsync_opts+=" --owner --group"
 318                 fi
 319                 cd "${TMPDIR}"/portage
 320                 rsync ${rsync_opts} . "${PORTDIR%%/}"
 321                 cd "${DISTDIR}"
 322
 323                 __vecho "Cleaning up ..."
 324                 rm -fr "${TMPDIR}"
 325         fi
 326
 327         if has metadata-transfer ${FEATURES} ; then
 328                 __vecho "Updating cache ..."
 329                 "${PORTAGE_BIN_PATH}/emerge" --metadata
 330         fi
 331         local post_sync=${PORTAGE_CONFIGROOT}etc/portage/bin/post_sync
 332         [ -x "${post_sync}" ] && "${post_sync}"
 333         # --quiet suppresses output if there are no relevant news items
 334         has news ${FEATURES} && "${PORTAGE_BIN_PATH}/emerge" --check-news --quiet
 335         return 0
 336 }
 337
 338 do_snapshot() {
 339         local ignore_timestamp="$1"
 340         local date="$2"
 341
 342         local r=1
 343
 344         local base_file="portage-${date}.tar"
 345
 346         local have_files=0
 347         local mirror
 348
 349         local compressions=""
 350         type -P bzcat > /dev/null && compressions="${compressions} bz2"
 351
 352         if [[ -z ${compressions} ]] ; then
 353                 eecho "unable to locate any decompressors (xzcat or bzcat or zcat)"
 354                 exit 1
 355         fi
 356
 357         for mirror in ${GENTOO_MIRRORS} ; do
 358
 359                 mirror=${mirror%/}
 360                 __vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."
 361
 362                 for compression in ${compressions} ; do
 363                         local file="portage-${date}.tar.${compression}"
 364                         local digest="${file}.md5sum"
 365                         local signature="${file}.gpgsig"
 366
 367                         if [ -s "${DISTDIR}/${file}" -a -s "${DISTDIR}/${digest}" -a -s "${DISTDIR}/${signature}" ] ; then
 368                                 check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
 369                                 check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
 370                                 have_files=1
 371                         fi
 372
 373                         if [ ${have_files} -eq 0 ] ; then
 374                                 fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
 375                                 fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
 376                                 fetch_file "${mirror}/snapshots/${file}" "${file}" && \
 377                                 check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
 378                                 check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
 379                                 have_files=1
 380                         fi
 381
 382                         #
 383                         # If timestamp is invalid
 384                         # we want to try and retrieve
 385                         # from a different mirror
 386                         #
 387                         if [ ${have_files} -eq 1 ]; then
 388
 389                                 __vecho "Getting snapshot timestamp ..."
 390                                 local snapshot_timestamp=$(get_snapshot_timestamp "${DISTDIR}/${file}")
 391
 392                                 if [ ${ignore_timestamp} == 0 ]; then
 393                                         if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
 394                                                 wecho "portage is newer than snapshot"
 395                                                 have_files=0
 396                                         fi
 397                                 else
 398                                         local utc_seconds=$(get_utc_second_from_string "${date}")
 399
 400                                         #
 401                                         # Check that this snapshot
 402                                         # is what it claims to be ...
 403                                         #
 404                                         if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
 405                                                 [ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
 406
 407                                                 wecho "snapshot timestamp is not in acceptable period"
 408                                                 have_files=0
 409                                         fi
 410                                 fi
 411                         fi
 412
 413                         if [ ${have_files} -eq 1 ]; then
 414                                 break
 415                         else
 416                                 #
 417                                 # Remove files and use a different mirror
 418                                 #
 419                                 rm -f "${DISTDIR}/${file}" "${DISTDIR}/${digest}" "${DISTDIR}/${signature}"
 420                         fi
 421                 done
 422
 423                 [ ${have_files} -eq 1 ] && break
 424         done
 425
 426         if [ ${have_files} -eq 1 ]; then
 427                 sync_local "${DISTDIR}/${file}" && r=0
 428         else
 429                 __vecho "${date} snapshot was not found"
 430         fi
 431
 432         return "${r}"
 433 }
 434
 435 do_latest_snapshot() {
 436         local attempts=0
 437         local r=1
 438
 439         __vecho "Fetching most recent snapshot ..."
 440
 441         # The snapshot for a given day is generated at 00:45 UTC on the following
 442         # day, so the current day's snapshot (going by UTC time) hasn't been
 443         # generated yet.  Therefore, always start by looking for the previous day's
 444         # snapshot (for attempts=1, subtract 1 day from the current UTC time).
 445
 446         # Timestamps that differ by less than 2 hours
 447         # are considered to be approximately equal.
 448         local min_time_diff=$(( 2 * 60 * 60 ))
 449
 450         local existing_timestamp=$(get_portage_timestamp)
 451         local timestamp_difference
 452         local timestamp_problem
 453         local approx_snapshot_time
 454         local start_time=$(get_utc_date_in_seconds)
 455         local start_hour=$(get_date_part ${start_time} "%H")
 456
 457         # Daily snapshots are created at 00:45 and are not
 458         # available until after 01:00. Don't waste time trying
 459         # to fetch a snapshot before it's been created.
 460         if [ ${start_hour} -lt 1 ] ; then
 461                 (( start_time -= 86400 ))
 462         fi
 463         local snapshot_date=$(get_date_part ${start_time} "%Y%m%d")
 464         local snapshot_date_seconds=$(get_utc_second_from_string ${snapshot_date})
 465
 466         while (( ${attempts} <  40 )) ; do
 467                 (( attempts++ ))
 468                 (( snapshot_date_seconds -= 86400 ))
 469                 # snapshots are created at 00:45
 470                 (( approx_snapshot_time = snapshot_date_seconds + 86400 + 2700 ))
 471                 (( timestamp_difference = existing_timestamp - approx_snapshot_time ))
 472                 [ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference ))
 473                 snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d")
 474
 475                 timestamp_problem=""
 476                 if [ ${timestamp_difference} -eq 0 ]; then
 477                         timestamp_problem="is identical to"
 478                 elif [ ${timestamp_difference} -lt ${min_time_diff} ]; then
 479                         timestamp_problem="is possibly identical to"
 480                 elif [ ${approx_snapshot_time} -lt ${existing_timestamp} ] ; then
 481                         timestamp_problem="is newer than"
 482                 fi
 483
 484                 if [ -n "${timestamp_problem}" ]; then
 485                         ewarn "Latest snapshot date: ${snapshot_date}"
 486                         ewarn
 487                         ewarn "Approximate snapshot timestamp: ${approx_snapshot_time}"
 488                         ewarn "       Current local timestamp: ${existing_timestamp}"
 489                         ewarn
 490                         echo -e "The current local timestamp" \
 491                                 "${timestamp_problem} the" \
 492                                 "timestamp of the latest" \
 493                                 "snapshot. In order to force sync," \
 494                                 "use the --revert option or remove" \
 495                                 "the timestamp file located at" \
 496                                 "'${PORTDIR}/metadata/timestamp.x'." | fmt -w 70 | \
 497                                 while read -r line ; do
 498                                         ewarn "${line}"
 499                                 done
 500                         r=0
 501                         break
 502                 fi
 503
 504                 if do_snapshot 0 "${snapshot_date}"; then
 505                         r=0
 506                         break;
 507                 fi
 508         done
 509
 510         return "${r}"
 511 }
 512
 513 fetch_from_mirrors() {
 514         local i URI FILE MIRRORS
 515         if [[ "$#" == 3 ]]; then
 516                 MIRRORS="${3}"
 517         else
 518                 MIRRORS=$GENTOO_MIRRORS
 519         fi
 520         FILE="$2"
 521         for i in $MIRRORS ; do
 522                 URI="${i%/}/${1#/}"
 523                 fetch_file "${URI}" "${FILE}" && return 0
 524         done
 525         return 1
 526 }
 527
 528 verify_md5_file() {
 529         local FILE MD5_LOC
 530         FILE="$1"
 531         if [[ $# == 2 ]]; then
 532                 MD5_LOC="$2"
 533         else
 534                 MD5_LOC="$(pwd)/$1.md5sum"
 535         fi
 536         check_file_digest "${MD5_LOC}" "${FILE}"
 537 }
 538
 539 #--------------------
 540 #inline actual script
 541 #--------------------
 542
 543 if ! type -p patcher &> /dev/null; then
 544         echo "!!!"
 545         echo "!!! cannot find patcher, did you emerge dev-util/diffball?"
 546         echo "!!! lack of patcher == have to do full fetch"
 547         echo "!!!"
 548         sleep 10
 549         if do_latest_snapshot; then
 550                 rm -fr "${TMPDIR}"
 551                 cleanse_state_dir
 552                 exit 0
 553         fi
 554         exit 1
 555 fi
 556
 557 echo "Looking for available base versions for a delta"
 558
 559 #note we're already in distdir
 560
 561 unset base_version
 562 # portage-snapshots in reverse order.
 563 # icky.
 564 unset dfile
 565 potentials="$(ls -1 portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 ${STATE_DIR}/portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 2> /dev/null | sed -e 's:^.*/::' | sort -r)"
 566 for basef in ${potentials}; do
 567         chksum=''
 568         found="dar"
 569         if [ -e "${STATE_DIR}/${basef}.md5sum" ]; then
 570                 chksum="${STATE_DIR}/${basef}.md5sum"
 571         elif [ -e "${basef}.md5sum" ]; then
 572                 chksum="${DISTDIR}/${basef}.md5sum"
 573         else
 574                 echo "attempting to get md5sum for $basef"
 575                 if ! fetch_from_mirrors "/snapshots/${basef}.md5sum" "${basef}.md5sum"; then
 576                         echo "can't get md5 for ${basef}"
 577                         continue
 578                 fi
 579                 chksum="${basef}.md5sum"
 580         fi
 581         if [ -e "${basef}" ]; then
 582                 dfile="${DISTDIR}/${basef}"
 583         else
 584                 dfile="${STATE_DIR}/${basef}"
 585         fi
 586         if ! verify_md5_file "${dfile}" "${chksum}"; then
 587                 echo "found a stale snapshot.  cleansing"
 588                 rm -f "${dfile}" &> /dev/null
 589                 rm -f "${chksum}.md5sum" &> /dev/null
 590                 dar=""
 591         else
 592                 base_version="${basef}"
 593                 break
 594         fi
 595 done
 596
 597 #by this point, we either have a base_version, or we don't.
 598 if [[ -z ${base_version} ]]; then
 599         echo "no base found.  resorting to pulling a full version"
 600         if do_latest_snapshot; then
 601                 rm -fr "${TMPDIR}"
 602                 cleanse_state_dir
 603                 exit 0
 604         fi
 605         exit 1
 606 fi
 607
 608 #we have a md5 verified base.  now we get the patch.
 609
 610 base_date="${base_version%.tar.bz2}"
 611 base_date="${base_date#portage-}"
 612 # we now have yyyymmdd
 613
 614 patches=''
 615 echo "fetching patches"
 616 fetched='asdf'
 617 while [[ -n ${fetched} ]]; do
 618         next_day=$(increment_date ${base_date} +1)
 619         # if we can't get a *single* patch or md5, even one missing, do full.
 620         p="snapshot-${base_date}-${next_day}.patch.bz2"
 621         if [[ ! -e ${p}.md5sum ]] && ! fetch_from_mirrors "/snapshots/deltas/${p}.md5sum" "${p}.md5sum"; then
 622                 echo "failed fetching ${p}.md5sum"
 623                 fetched=''
 624                 break
 625         fi
 626         fetch="yes"
 627         if [[ -e ${p} ]]; then
 628                 if ! verify_md5_file "${p}"; then
 629                         rm -f "${p}" &> /dev/null
 630                 else
 631                         fetch=""
 632                 fi
 633         fi
 634         if [[ -n $fetch ]]; then
 635                 if ! fetch_from_mirrors "/snapshots/deltas/${p}" "${p}"; then
 636                         echo "failed fetching ${p}"
 637                         fetched=''
 638                 fi
 639         fi
 640         if [[ -z ${fetched} ]]; then
 641                 break
 642         fi
 643         if ! verify_md5_file "${p}"; then
 644                 echo "md5 failed on ${p}"
 645                 fetched=''
 646                 break
 647         fi
 648         patches="${patches} ${p}"
 649         base_date="${next_day}"
 650 done
 651 final_date=${base_date}
 652
 653 if [[ -z $patches ]]; then
 654         echo "no patches found? up to date?"
 655         if [[ -n $MUST_SYNC ]]; then
 656                 echo "syncing with existing file"
 657                 if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 &&
 658                         ! -e ${DISTDIR}/portage-${base_date}.tar.bz2.gpgsig ]] && \
 659                         ! fetch_from_mirrors "/snapshots/portage-${base_date}.tar.bz2.gpgsig" "portage-${base_date}.tar.bz2.gpgsig" ; then
 660                         eecho "Couldn't fetch portage-${base_date}.tar.bz2.gpgsig"
 661                         exit 5
 662                 fi
 663                 if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
 664                         check_file_signature "${DISTDIR}/portage-${base_date}.tar.bz2.gpgsig" "${dfile}" || exit 1
 665                 fi
 666                 sync_local "${dfile}" && rm -fr "${TMPDIR}"
 667         else
 668                 rm -fr "${TMPDIR}"
 669         fi
 670         exit $?
 671 fi
 672
 673 unset got_umd5
 674 #grab the md5 for later usage.
 675 if [[ ! -e portage-${final_date}.tar.bz2.md5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.md5sum" "portage-${final_date}.tar.bz2.md5sum"; then
 676         echo "warning... couldn't grab the md5sum for ${final_date}.  which is odd"
 677         echo "thus, bailing (sorry)"
 678         exit 5
 679 else
 680         if [[ ! -e portage-${final_date}.tar.bz2.umd5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then
 681                 if ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then
 682                         echo "couldn't grab umd5sum (uncompressed md5sum) for ${final_date}."
 683                         echo "can't compensate for bzip2 version differences iow."
 684                 else
 685                         got_umd5=1
 686                 fi
 687         else
 688                 got_umd5=1
 689         fi
 690 fi
 691
 692 if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 && ! -e portage-${final_date}.tar.bz2.gpgsig ]] && \
 693         ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.gpgsig" "portage-${final_date}.tar.bz2.gpgsig" ; then
 694         echo "warning... couldn't grab the gpgsig for ${final_date}.  which is odd"
 695         echo "thus, bailing (sorry)"
 696         exit 5
 697 fi
 698
 699 # got our patches.
 700 if ! patcher -v "${dfile}" ${patches} "${TMPDIR}/portage-${final_date}.tar"; then
 701         echo "reconstruction failed (contact the author with the error from the reconstructor please)"
 702         rm -f "${TMPDIR}/portage-${final_date}.tar"
 703         if do_latest_snapshot; then
 704                 rm -fr "${TMPDIR}"
 705                 cleanse_state_dir
 706                 exit 0
 707         fi
 708         exit 1
 709 fi
 710 verified=0
 711 if [[ -n $got_umd5 ]]; then
 712         echo "verifying uncompressed md5"
 713         if ! verify_md5_file "${TMPDIR}/portage-${final_date}.tar" "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum"; then
 714                 echo "uncompressed verification failed.  This means either you found a bug in diffball, or something odd is going on"
 715                 echo "with upstream patch generation"
 716                 echo "trying md5sum next, which probably will fail."
 717         else
 718                 verified="1"
 719         fi
 720 fi
 721
 722 unset need_last_sync
 723 if [ "$verified" == "1" ]; then
 724         need_last_sync="dar"
 725         if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
 726                 # BUG: Signature verification will fail if the local bzip2
 727                 # program does not produce output that is perfectly identical
 728                 # to the bzip2 program used to compress the signed tar file.
 729                 echo "recompressing ..."
 730                 bzip2 -vk9 "${TMPDIR}/portage-${final_date}.tar"
 731                 check_file_signature "${DISTDIR}/portage-${final_date}.tar.bz2.gpgsig" "${TMPDIR}/portage-${final_date}.tar.bz2" || exit 1
 732         else
 733                 echo "recompressing. (backgrounding)"
 734                 bzip2 -vk9 "${TMPDIR}/portage-${final_date}.tar" &
 735         fi
 736
 737         echo "beginning update to the tree"
 738         sync_local "${TMPDIR}/portage-${final_date}.tar"
 739         echo "doing final md5 stuff"
 740         wait
 741         # bzip2 is finished now.
 742         rm -f "${TMPDIR}/portage-${final_date}.tar"
 743 else
 744         echo "recompressing."
 745         bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar.bz2"
 746 fi
 747
 748 echo "verifying generated tarball"
 749
 750 if ! verify_md5_file "${TMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/portage-${final_date}.tar.bz2.md5sum"; then
 751         if [[ -z $verified ]]; then
 752                 echo "couldn't verify the generated tarball.  bug, most likely."
 753                 exit 5
 754         fi
 755         # hokay.  md5 doesn't agree with umd5. bzip2 issue in effect.
 756         echo "compressed md5 differs, but uncompressed md5 says it right.  bzip2 version incompatability in other words"
 757         echo "saving the md5"
 758         if type -p md5sum &> /dev/null; then
 759                 md5sum "${TMPDIR}/portage-${final_date}.tar.bz2" | sed -e "s:${TMPDIR}/\?::" > \
 760                         "${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum"
 761         elif type -p md5 &> /dev/null; then
 762                 echo "$(md5 -q "${TMPDIR}/portage-${final_date}.tar.bz2")  portage-${final_date}.tar.bz2" > \
 763                         "${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum"
 764         else
 765                 echo "couldn't find either md5 or md5sum.  something is screwed... (bailing, sorry)"
 766                 exit 7
 767         fi
 768         mv "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum" "${TMPDIR}/portage-${final_date}.tar.bz2" "${STATE_DIR}/"
 769         dfile="${STATE_DIR}/portage-${final_date}.tar.bz2"
 770 else
 771         dfile="${DISTDIR}/portage-${final_date}.tar.bz2"
 772         mv "${TMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/"
 773 fi
 774
 775 if [ -z "${need_last_sync}" ]; then
 776         if [[ ${WEBSYNC_VERIFY_SIGNATURE} == 1 ]] ; then
 777                 check_file_signature "${DISTDIR}/portage-${final_date}.tar.bz2.gpgsig" "${dfile}" || exit 1
 778         fi
 779         echo "beginning update to the tree"
 780         sync_local "${dfile}"
 781 fi
 782
 783 if [[ -z $KEEP_OLDIES ]]; then
 784         echo "cleansing"
 785         for x in $potentials; do
 786                 echo "removing ${x}"
 787                 rm -f "${DISTDIR}/${x}"{,.md5sum,.umd5sum,.gpgsig} &> /dev/null
 788                 rm -f "${STATE_DIR}/${x}"{,.md5sum,.umd5sum} &> /dev/null
 789         done
 790 fi
 791 rm -rf "${TMPDIR}"
 792 echo "done."
 793