bin/emerge-webrsync

   1 #!/bin/bash
   2 # Copyright 1999-2013 Gentoo Foundation
   3 # Distributed under the terms of the GNU General Public License v2
   4 # Author: Karl Trygve Kalleberg <karltk@gentoo.org>
   5 # Rewritten from the old, Perl-based emerge-webrsync script
   6 # Author: Alon Bar-Lev <alon.barlev@gmail.com>
   7 # Major rewrite from Karl's scripts.
   8
   9 # TODO:
  10 #  - all output should prob be converted to e* funcs
  11 #  - add support for ROOT
  12
  13 #
  14 # gpg key import
  15 # KEY_ID=0x96D8BF6D
  16 # gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
  17 # gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
  18 #
  19
  20 # Only echo if in verbose mode
  21 vvecho() { [[ ${do_verbose} -eq 1 ]] && echo "$@" ; }
  22 # Only echo if not in verbose mode
  23 nvecho() { [[ ${do_verbose} -eq 0 ]] && echo "$@" ; }
  24 # warning echos
  25 wecho() { echo "${argv0##*/}: warning: $*" 1>&2 ; }
  26 # error echos
  27 eecho() { echo "${argv0##*/}: error: $*" 1>&2 ; }
  28
  29 argv0=$0
  30
  31 # Use portageq from the same directory/prefix as the current script, so
  32 # that we don't have to rely on PATH including the current EPREFIX.
  33 scriptpath=${BASH_SOURCE[0]}
  34 if [ -x "${scriptpath%/*}/portageq" ]; then
  35         portageq=${scriptpath%/*}/portageq
  36 elif type -P portageq > /dev/null ; then
  37         portageq=portageq
  38 else
  39         eecho "could not find 'portageq'; aborting"
  40         exit 1
  41 fi
  42 eval "$("${portageq}" envvar -v DISTDIR EPREFIX FEATURES \
  43         FETCHCOMMAND GENTOO_MIRRORS \
  44         PORTAGE_BIN_PATH PORTAGE_CONFIGROOT PORTAGE_GPG_DIR \
  45         PORTAGE_NICENESS PORTAGE_REPOSITORIES PORTAGE_RSYNC_EXTRA_OPTS \
  46         PORTAGE_RSYNC_OPTS PORTAGE_TMPDIR \
  47         USERLAND http_proxy ftp_proxy)"
  48 export http_proxy ftp_proxy
  49
  50 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
  51
  52 repo_name=gentoo
  53 repo_location=$(__repo_key "${repo_name}" location)
  54 if [[ -z ${repo_location} ]]; then
  55         eecho "Repository '${repo_name}' not found"
  56         exit 1
  57 fi
  58 repo_sync_type=$(__repo_key "${repo_name}" sync-type)
  59
  60 # If PORTAGE_NICENESS is overriden via the env then it will
  61 # still pass through the portageq call and override properly.
  62 if [ -n "${PORTAGE_NICENESS}" ]; then
  63         renice $PORTAGE_NICENESS $$ > /dev/null
  64 fi
  65
  66 do_verbose=0
  67 do_debug=0
  68 keep=false
  69
  70 if has webrsync-gpg ${FEATURES} ; then
  71         WEBSYNC_VERIFY_SIGNATURE=1
  72 else
  73         WEBSYNC_VERIFY_SIGNATURE=0
  74 fi
  75 if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
  76         eecho "please set PORTAGE_GPG_DIR in make.conf"
  77         exit 1
  78 fi
  79
  80 do_tar() {
  81         local file=$1; shift
  82         local decompressor
  83         case ${file} in
  84                 *.xz)   decompressor="xzcat" ;;
  85                 *.bz2)  decompressor="bzcat" ;;
  86                 *.gz)   decompressor="zcat"  ;;
  87                 *)      decompressor="cat"   ;;
  88         esac
  89         ${decompressor} "${file}" | tar "$@"
  90         _pipestatus=${PIPESTATUS[*]}
  91         [[ ${_pipestatus// /} -eq 0 ]]
  92 }
  93
  94 get_utc_date_in_seconds() {
  95         date -u +"%s"
  96 }
  97
  98 get_date_part() {
  99         local utc_time_in_secs="$1"
 100         local part="$2"
 101
 102         if      [[ ${USERLAND} == BSD ]] ; then
 103                 date -r ${utc_time_in_secs} -u +"${part}"
 104         else
 105                 date -d @${utc_time_in_secs} -u +"${part}"
 106         fi
 107 }
 108
 109 get_utc_second_from_string() {
 110         local s="$1"
 111         if [[ ${USERLAND} == BSD ]] ; then
 112                 # Specify zeros for the least significant digits, or else those
 113                 # digits are inherited from the current system clock time.
 114                 date -juf "%Y%m%d%H%M.%S" "${s}0000.00" +"%s"
 115         else
 116                 date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s"
 117         fi
 118 }
 119
 120 get_portage_timestamp() {
 121         local portage_current_timestamp=0
 122
 123         if [ -f "${repo_location}/metadata/timestamp.x" ]; then
 124                 portage_current_timestamp=$(cut -f 1 -d " " "${repo_location}/metadata/timestamp.x" )
 125         fi
 126
 127         echo "${portage_current_timestamp}"
 128 }
 129
 130 fetch_file() {
 131         local URI="$1"
 132         local FILE="$2"
 133         local opts
 134
 135         if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
 136                 opts="--continue $(nvecho -q)"
 137         elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
 138                 opts="--continue-at - $(nvecho -s -f)"
 139         else
 140                 rm -f "${DISTDIR}/${FILE}"
 141         fi
 142
 143         __vecho "Fetching file ${FILE} ..."
 144         # already set DISTDIR=
 145         eval "${FETCHCOMMAND} ${opts}"
 146         if [[ $? -eq 0 && -s ${DISTDIR}/${FILE} ]] ; then
 147                 return 0
 148         else
 149                 rm -f "${DISTDIR}/${FILE}"
 150                 return 1
 151         fi
 152 }
 153
 154 check_file_digest() {
 155         local digest="$1"
 156         local file="$2"
 157         local r=1
 158
 159         __vecho "Checking digest ..."
 160
 161         if type -P md5sum > /dev/null; then
 162                 local md5sum_output=$(md5sum "${file}")
 163                 local digest_content=$(< "${digest}")
 164                 [ "${md5sum_output%%[[:space:]]*}" = "${digest_content%%[[:space:]]*}" ] && r=0
 165         elif type -P md5 > /dev/null; then
 166                 [ "$(md5 -q "${file}")" == "$(cut -d ' ' -f 1 "${digest}")" ] && r=0
 167         else
 168                 eecho "cannot check digest: no suitable md5/md5sum binaries found"
 169         fi
 170
 171         return "${r}"
 172 }
 173
 174 check_file_signature() {
 175         local signature="$1"
 176         local file="$2"
 177         local r=1
 178
 179         if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then
 180
 181                 __vecho "Checking signature ..."
 182
 183                 if type -P gpg > /dev/null; then
 184                         gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
 185                 else
 186                         eecho "cannot check signature: gpg binary not found"
 187                         exit 1
 188                 fi
 189         else
 190                 r=0
 191         fi
 192
 193         return "${r}"
 194 }
 195
 196 get_snapshot_timestamp() {
 197         local file="$1"
 198
 199         do_tar "${file}" --to-stdout -xf - portage/metadata/timestamp.x | cut -f 1 -d " "
 200 }
 201
 202 sync_local() {
 203         local file="$1"
 204
 205         __vecho "Syncing local tree ..."
 206
 207         local ownership="portage:portage"
 208         if has usersync ${FEATURES} ; then
 209                 case "${USERLAND}" in
 210                         BSD)
 211                                 ownership=$(stat -f '%Su:%Sg' "${repo_location}")
 212                                 ;;
 213                         *)
 214                                 ownership=$(stat -c '%U:%G' "${repo_location}")
 215                                 ;;
 216                 esac
 217         fi
 218
 219         if type -P tarsync > /dev/null ; then
 220                 local chown_opts="-o ${ownership%:*} -g ${ownership#*:}"
 221                 chown ${ownership} "${repo_location}" > /dev/null 2>&1 || chown_opts=""
 222                 if ! tarsync $(vvecho -v) -s 1 ${chown_opts} \
 223                         -e /distfiles -e /packages -e /local "${file}" "${repo_location}"; then
 224                         eecho "tarsync failed; tarball is corrupt? (${file})"
 225                         return 1
 226                 fi
 227         else
 228                 if ! do_tar "${file}" xf -; then
 229                         eecho "tar failed to extract the image. tarball is corrupt? (${file})"
 230                         rm -fr portage
 231                         return 1
 232                 fi
 233
 234                 # Free disk space
 235                 ${keep} || rm -f "${file}"
 236
 237                 local rsync_opts="${PORTAGE_RSYNC_OPTS} ${PORTAGE_RSYNC_EXTRA_OPTS}"
 238                 if chown ${ownership} portage > /dev/null 2>&1; then
 239                         chown -R ${ownership} portage
 240                         rsync_opts+=" --owner --group"
 241                 fi
 242                 cd portage
 243                 rsync ${rsync_opts} . "${repo_location%%/}"
 244                 cd ..
 245
 246                 __vecho "Cleaning up ..."
 247                 rm -fr portage
 248         fi
 249
 250         if has metadata-transfer ${FEATURES} ; then
 251                 __vecho "Updating cache ..."
 252                 "${PORTAGE_BIN_PATH}/emerge" --metadata
 253         fi
 254         local post_sync=${PORTAGE_CONFIGROOT}etc/portage/bin/post_sync
 255         [ -x "${post_sync}" ] && "${post_sync}"
 256         # --quiet suppresses output if there are no relevant news items
 257         has news ${FEATURES} && "${PORTAGE_BIN_PATH}/emerge" --check-news --quiet
 258         return 0
 259 }
 260
 261 do_snapshot() {
 262         local ignore_timestamp="$1"
 263         local date="$2"
 264
 265         local r=1
 266
 267         local base_file="portage-${date}.tar"
 268
 269         local have_files=0
 270         local mirror
 271
 272         local compressions=""
 273         # xz is not supported in app-arch/tarsync, so use
 274         # bz2 format if we have tarsync.
 275         if ! type -P tarsync > /dev/null ; then
 276                 type -P xzcat > /dev/null && compressions="${compressions} xz"
 277         fi
 278         type -P bzcat > /dev/null && compressions="${compressions} bz2"
 279         type -P  zcat > /dev/null && compressions="${compressions} gz"
 280         if [[ -z ${compressions} ]] ; then
 281                 eecho "unable to locate any decompressors (xzcat or bzcat or zcat)"
 282                 exit 1
 283         fi
 284
 285         for mirror in ${GENTOO_MIRRORS} ; do
 286
 287                 mirror=${mirror%/}
 288                 __vecho "Trying to retrieve ${date} snapshot from ${mirror} ..."
 289
 290                 for compression in ${compressions} ; do
 291                         local file="portage-${date}.tar.${compression}"
 292                         local digest="${file}.md5sum"
 293                         local signature="${file}.gpgsig"
 294
 295                         if [ -s "${DISTDIR}/${file}" -a -s "${DISTDIR}/${digest}" -a -s "${DISTDIR}/${signature}" ] ; then
 296                                 check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
 297                                 check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
 298                                 have_files=1
 299                         fi
 300
 301                         if [ ${have_files} -eq 0 ] ; then
 302                                 fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
 303                                 fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
 304                                 fetch_file "${mirror}/snapshots/${file}" "${file}" && \
 305                                 check_file_digest "${DISTDIR}/${digest}" "${DISTDIR}/${file}" && \
 306                                 check_file_signature "${DISTDIR}/${signature}" "${DISTDIR}/${file}" && \
 307                                 have_files=1
 308                         fi
 309
 310                         #
 311                         # If timestamp is invalid
 312                         # we want to try and retrieve
 313                         # from a different mirror
 314                         #
 315                         if [ ${have_files} -eq 1 ]; then
 316
 317                                 __vecho "Getting snapshot timestamp ..."
 318                                 local snapshot_timestamp=$(get_snapshot_timestamp "${DISTDIR}/${file}")
 319
 320                                 if [ ${ignore_timestamp} == 0 ]; then
 321                                         if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
 322                                                 wecho "portage is newer than snapshot"
 323                                                 have_files=0
 324                                         fi
 325                                 else
 326                                         local utc_seconds=$(get_utc_second_from_string "${date}")
 327
 328                                         #
 329                                         # Check that this snapshot
 330                                         # is what it claims to be ...
 331                                         #
 332                                         if [ ${snapshot_timestamp} -lt ${utc_seconds} ] || \
 333                                                 [ ${snapshot_timestamp} -gt $((${utc_seconds}+ 2*86400)) ]; then
 334
 335                                                 wecho "snapshot timestamp is not in acceptable period"
 336                                                 have_files=0
 337                                         fi
 338                                 fi
 339                         fi
 340
 341                         if [ ${have_files} -eq 1 ]; then
 342                                 break
 343                         else
 344                                 #
 345                                 # Remove files and use a different mirror
 346                                 #
 347                                 rm -f "${DISTDIR}/${file}" "${DISTDIR}/${digest}" "${DISTDIR}/${signature}"
 348                         fi
 349                 done
 350
 351                 [ ${have_files} -eq 1 ] && break
 352         done
 353
 354         if [ ${have_files} -eq 1 ]; then
 355                 sync_local "${DISTDIR}/${file}" && r=0
 356         else
 357                 __vecho "${date} snapshot was not found"
 358         fi
 359
 360         ${keep} || rm -f "${DISTDIR}/${file}" "${DISTDIR}/${digest}" "${DISTDIR}/${signature}"
 361         return "${r}"
 362 }
 363
 364 do_latest_snapshot() {
 365         local attempts=0
 366         local r=1
 367
 368         __vecho "Fetching most recent snapshot ..."
 369
 370         # The snapshot for a given day is generated at 00:45 UTC on the following
 371         # day, so the current day's snapshot (going by UTC time) hasn't been
 372         # generated yet.  Therefore, always start by looking for the previous day's
 373         # snapshot (for attempts=1, subtract 1 day from the current UTC time).
 374
 375         # Timestamps that differ by less than 2 hours
 376         # are considered to be approximately equal.
 377         local min_time_diff=$(( 2 * 60 * 60 ))
 378
 379         local existing_timestamp=$(get_portage_timestamp)
 380         local timestamp_difference
 381         local timestamp_problem
 382         local approx_snapshot_time
 383         local start_time=$(get_utc_date_in_seconds)
 384         local start_hour=$(get_date_part ${start_time} "%H")
 385
 386         # Daily snapshots are created at 00:45 and are not
 387         # available until after 01:00. Don't waste time trying
 388         # to fetch a snapshot before it's been created.
 389         if [ ${start_hour} -lt 1 ] ; then
 390                 (( start_time -= 86400 ))
 391         fi
 392         local snapshot_date=$(get_date_part ${start_time} "%Y%m%d")
 393         local snapshot_date_seconds=$(get_utc_second_from_string ${snapshot_date})
 394
 395         while (( ${attempts} <  40 )) ; do
 396                 (( attempts++ ))
 397                 (( snapshot_date_seconds -= 86400 ))
 398                 # snapshots are created at 00:45
 399                 (( approx_snapshot_time = snapshot_date_seconds + 86400 + 2700 ))
 400                 (( timestamp_difference = existing_timestamp - approx_snapshot_time ))
 401                 [ ${timestamp_difference} -lt 0 ] && (( timestamp_difference = -1 * timestamp_difference ))
 402                 snapshot_date=$(get_date_part ${snapshot_date_seconds} "%Y%m%d")
 403
 404                 timestamp_problem=""
 405                 if [ ${timestamp_difference} -eq 0 ]; then
 406                         timestamp_problem="is identical to"
 407                 elif [ ${timestamp_difference} -lt ${min_time_diff} ]; then
 408                         timestamp_problem="is possibly identical to"
 409                 elif [ ${approx_snapshot_time} -lt ${existing_timestamp} ] ; then
 410                         timestamp_problem="is newer than"
 411                 fi
 412
 413                 if [ -n "${timestamp_problem}" ]; then
 414                         ewarn "Latest snapshot date: ${snapshot_date}"
 415                         ewarn
 416                         ewarn "Approximate snapshot timestamp: ${approx_snapshot_time}"
 417                         ewarn "       Current local timestamp: ${existing_timestamp}"
 418                         ewarn
 419                         echo -e "The current local timestamp" \
 420                                 "${timestamp_problem} the" \
 421                                 "timestamp of the latest" \
 422                                 "snapshot. In order to force sync," \
 423                                 "use the --revert option or remove" \
 424                                 "the timestamp file located at" \
 425                                 "'${repo_location}/metadata/timestamp.x'." | fmt -w 70 | \
 426                                 while read -r line ; do
 427                                         ewarn "${line}"
 428                                 done
 429                         r=0
 430                         break
 431                 fi
 432
 433                 if do_snapshot 0 "${snapshot_date}"; then
 434                         r=0
 435                         break;
 436                 fi
 437         done
 438
 439         return "${r}"
 440 }
 441
 442 usage() {
 443         cat <<-EOF
 444         Usage: $0 [options]
 445
 446         Options:
 447           --revert=yyyymmdd   Revert to snapshot
 448           -k, --keep          Keep snapshots in DISTDIR (don't delete)
 449           -q, --quiet         Only output errors
 450           -v, --verbose       Enable verbose output
 451           -x, --debug         Enable debug output
 452           -h, --help          This help screen (duh!)
 453         EOF
 454         if [[ -n $* ]] ; then
 455                 printf "\nError: %s\n" "$*" 1>&2
 456                 exit 1
 457         else
 458                 exit 0
 459         fi
 460 }
 461
 462 main() {
 463         local arg
 464         local revert_date
 465
 466         for arg in "$@" ; do
 467                 local v=${arg#*=}
 468                 case ${arg} in
 469                         -h|--help)    usage ;;
 470                         -k|--keep)    keep=true ;;
 471                         -q|--quiet)   PORTAGE_QUIET=1 ;;
 472                         -v|--verbose) do_verbose=1 ;;
 473                         -x|--debug)   do_debug=1 ;;
 474                         --revert=*)   revert_date=${v} ;;
 475                         *)            usage "Invalid option '${arg}'" ;;
 476                 esac
 477         done
 478
 479         [[ -d ${repo_location} ]] || mkdir -p "${repo_location}"
 480         if [[ ! -w ${repo_location} ]] ; then
 481                 eecho "Repository '${repo_name}' is not writable: ${repo_location}"
 482                 exit 1
 483         fi
 484
 485         [[ -d ${PORTAGE_TMPDIR}/portage ]] || mkdir -p "${PORTAGE_TMPDIR}/portage"
 486         TMPDIR=$(mktemp -d "${PORTAGE_TMPDIR}/portage/webrsync-XXXXXX")
 487         if [[ ! -w ${TMPDIR} ]] ; then
 488                 eecho "TMPDIR is not writable: ${TMPDIR}"
 489                 exit 1
 490         fi
 491         trap 'cd / ; rm -rf "${TMPDIR}"' EXIT
 492         cd "${TMPDIR}" || exit 1
 493
 494         ${keep} || DISTDIR=${TMPDIR}
 495         [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
 496
 497         if ${keep} && [[ ! -w ${DISTDIR} ]] ; then
 498                 eecho "DISTDIR is not writable: ${DISTDIR}"
 499                 exit 1
 500         fi
 501
 502         # This is a sanity check to help prevent people like funtoo users
 503         # from accidentally wiping out their git tree.
 504         if [[ -n ${repo_sync_type} && ${repo_sync_type} != rsync ]] ; then
 505                 echo "The current sync-type attribute of repository 'gentoo' is not set to 'rsync':" >&2
 506                 echo >&2
 507                 echo "  sync-type=${repo_sync_type}" >&2
 508                 echo >&2
 509                 echo "If you intend to use emerge-webrsync then please" >&2
 510                 echo "adjust sync-type and sync-uri attributes to refer to rsync." >&2
 511                 echo "emerge-webrsync exiting due to abnormal sync-type setting." >&2
 512                 exit 1
 513         fi
 514
 515         [[ ${do_debug} -eq 1 ]] && set -x
 516
 517         if [[ -n ${revert_date} ]] ; then
 518                 do_snapshot 1 "${revert_date}"
 519         else
 520                 do_latest_snapshot
 521         fi
 522 }
 523
 524 main "$@"