# $Id$
# Author: Karl Trygve Kalleberg <karltk@gentoo.org>
# Rewritten from the old, Perl-based emerge-webrsync script
+# Author: Alon Bar-Lev <alon.barlev@gmail.com>
+# Major rewrite from Karl's scripts.
+
+#
+# gpg key import
+# KEY_ID=0x7DDAD20D
+# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID
+# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust
+#
type portageq > /dev/null || exit $?
eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \
- PORTAGE_BIN_PATH PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \
- PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS http_proxy ftp_proxy EPREFIX)
+ PORTAGE_BIN_PATH PORTAGE_GPG_DIR PORTAGE_INST_UID PORTAGE_INST_GID \
+ PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \
+ http_proxy ftp_proxy EPREFIX)
DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync"
export http_proxy ftp_proxy
source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1
-if [ ! -d $DISTDIR ] ; then
- mkdir -p $DISTDIR
-fi
+# ${USERLAND} is unreliable since the portage tree might be empty, so test
+# success of the -r option to distinguish between gnu and bsd date.
+date -r 0 >&/dev/null && DATE_ARGS="BSD" || DATE_ARGS="GNU"
-cd "$DISTDIR"
-
-found=0
-if [ "$1" == "-v" ] ; then
- wgetops=
-else
- #this sucks. probably better to do 1> /dev/null
- #that said, waiting on the refactoring.
- if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then
- wgetops="-q"
- elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then
- wgetops="-s -f"
- fi
-fi
+do_verbose=0
-if type -P md5sum > /dev/null; then
- md5_com='md5sum -c "${FILE}.md5sum"'
-elif type -P md5 > /dev/null; then
- md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \ -f 1 ${FILE}.md5sum)" ]'
+if hasq webrsync-gpg ${FEATURES} ; then
+ WEBSYNC_VERIFY_SIGNATURE=1
else
- echo "warning, unable to do md5 verification of the snapshot!"
- echo "no suitable md5/md5sum binary was found!"
- md5_com='true'
+ WEBSYNC_VERIFY_SIGNATURE=0
fi
+if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then
+ echo "Error: Please set PORTAGE_GPG_DIR in make.conf"
+ exit 1
+fi
+
+get_utc_date_in_seconds() {
+ date -u +"%s"
+}
+
+get_date_part() {
+ local utc_time_in_secs="$1"
+ local part="$2"
+
+ if [ "${DATE_ARGS}" = "BSD" ]; then
+ date -r ${utc_time_in_secs} -u +"${part}"
+ else
+ date -d @${utc_time_in_secs} -u +"${part}"
+ fi
+}
+
+get_utc_from_string() {
+ local s="$1"
+
+ seconds=$(date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s")
+}
+
+get_portage_timestamp() {
+ local portage_current_timestamp=0
+
+ if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then
+ portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" )
+ fi
+
+ echo "${portage_current_timestamp}"
+}
+
+fetch_file() {
+ local URI="$1"
+ local FILE="$2"
+ local opts
+
+ if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then
+ opts="--continue"
+
+ [ "${do_verbose}" == 0 ] && opts="$opts -q"
+ elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then
+ opts="--continue-at -"
+
+ [ "${do_verbose}" == 0 ] && opts="$opts -s -f"
+ else
+ rm -f "${FILE}"
+ fi
+
+ echo "Fetching file ${FILE}..."
+
+ #already set DISTDIR=
+ if [ "${do_verbose}" == 0 ] ; then
+ eval "${FETCHCOMMAND}" ${opts} > /dev/null && [ -s "${FILE}" ]
+ else
+ eval "${FETCHCOMMAND}" ${opts} && [ -s "${FILE}" ]
+ fi
+}
+
+check_file_digest() {
+ local digest="$1"
+ local file="$2"
+ local r=1
+
+ echo "Checking digest..."
+
+ if type -P md5sum > /dev/null; then
+ md5sum -c $digest && r=0
+ elif type -P md5 > /dev/null; then
+ [ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0
+ else
+ echo "Error: Cannot check digest"
+ echo "No suitable md5/md5sum binaries found"
+ fi
+
+ return "${r}"
+}
+
+check_file_signature() {
+ local signature="$1"
+ local file="$2"
+ local r=1
+
+ if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then
+
+ echo "Checking signature..."
+
+ if type -p gpg > /dev/null; then
+ gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0
+ else
+ echo "Error: Cannot check signature"
+ echo "gpg binary not found"
+ fi
+ else
+ r=0
+ fi
+
+ return "${r}"
+}
+
+get_snapshot_timestamp() {
+ local file="$1"
+
+ tar --to-stdout -xf "${file}" portage/metadata/timestamp.x | cut -f 1 -d " "
+}
sync_local() {
- echo Syncing local tree...
+ local file="$1"
+
+ echo "Syncing local tree..."
+
if type -P tarsync &> /dev/null; then
- # tarsync doesn't take numeric uid/gid so we need to convert them.
- local inst_user="$(python -c "import pwd; print pwd.getpwuid(int('${PORTAGE_INST_UID:-0}'))[0]")"
- local inst_group="$(python -c "import grp; print grp.getgrgid(int('${PORTAGE_INST_GID:-0}'))[0]")"
- if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o ${inst_user} -g ${inst_group} -e /distfiles -e /packages -e /local; then
- echo "tarsync failed; tarball is corrupt?"
- exit 1;
+ if [ "${do_verbose}" != 0 ] ; then
+ tarsync_verbose=-v
+ fi
+ if ! tarsync $tarsync_verbose -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then
+ echo "Error: tarsync failed; tarball is corrupt? (${file})"
+ return 1
fi
- rm "${FILE}"
else
- if ! tar jxf $FILE; then
- echo "Tar failed to extract the image. Please review the output."
- echo "Executed command: tar jxf $FILE"
- exit 1
+ echo "Note: tarsync was not found, you may consider emerge it..."
+
+ if ! tar jxf "${file}"; then
+ echo "Error: tar failed to extract the image. tarball is corrupt? (${file})"
+ rm -fr portage
+ return 1
fi
- rm -f $FILE
+
+ # Free disk space
+ rm -f "${file}"
+
# Make sure user and group file ownership is ${PORTAGE_INST_UID}:${PORTAGE_INST_GID}
chown -R ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} portage
cd portage
rsync -av --progress --stats --delete --delete-after \
- --exclude='/distfiles' --exclude='/packages' \
- --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . ${PORTDIR%%/}
+ --exclude='/distfiles' --exclude='/packages' \
+ --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}"
cd ..
- echo "cleaning up"
- rm -rf portage
+
+ echo "Cleaning up..."
+ rm -fr portage
fi
+
if hasq metadata-transfer ${FEATURES} ; then
- echo "transferring metadata/cache"
+ echo "Updating cache..."
emerge --metadata
fi
[ -x "${EPREFIX}"/etc/portage/bin/post_sync ] && "${EPREFIX}"/etc/portage/bin/post_sync
}
-echo "Fetching most recent snapshot"
-
-declare -i attempts=0
-while (( $attempts < 40 )) ; do
- attempts=$(( attempts + 1 ))
-
- # The snapshot for a given day is generated at 01:45 UTC on the following
- # day, so the current day's snapshot (going by UTC time) hasn't been
- # generated yet. Therefore, always start by looking for the previous day's
- # snapshot (for attempts=1, subtract 1 day from the current UTC time).
- daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts})
- DATE_ARGS="-d @${daysbefore}"
- # ${USERLAND} is unreliable since the portage tree might be empty, so test
- # success of the -r option to distinguish between gnu and bsd date.
- date -r ${daysbefore} >&/dev/null && DATE_ARGS="-r ${daysbefore}"
- day=$(date ${DATE_ARGS} -u +"%d")
- month=$(date ${DATE_ARGS} -u +"%m")
- year=$(date ${DATE_ARGS} -u +"%Y")
-
- FILE_ORIG="portage-${year}${month}${day}.tar.bz2"
-
- echo "Attempting to fetch file dated: ${year}${month}${day}"
-
- got_md5=0
-
- if [ ! -e "${FILE_ORIG}.md5sum" ]; then
- FILE="${FILE_ORIG}.md5sum"
- for i in $GENTOO_MIRRORS ; do
- URI="${i}/snapshots/${FILE}"
- if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then
- got_md5=1
- break
+do_snapshot() {
+ local ignore_timestamp="$1"
+ local date="$2"
+
+ local r=1
+
+ local file="portage-${date}.tar.bz2"
+ local digest="${file}.md5sum"
+ local signature="${file}.gpgsig"
+
+ local have_files=0
+ local mirror
+
+ echo "Trying to retrieve ${date} snapshot..."
+
+ for mirror in ${GENTOO_MIRRORS} ; do
+
+ [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] && \
+ check_file_digest "${digest}" "${file}" && \
+ check_file_signature "${signature}" "${file}" && \
+ have_files=1
+
+ [ ${have_files} == 0 ] && \
+ fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \
+ fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \
+ fetch_file "${mirror}/snapshots/${file}" "${file}" && \
+ check_file_digest "${digest}" "${file}" && \
+ check_file_signature "${signature}" "${file}" && \
+ have_files=1
+
+ #
+ # If timestamp is invalid
+ # we want to try and retieve
+ # from a different mirror
+ #
+ if [ ${have_files} != 0 ]; then
+
+ echo "Getting snapshot timetasmp..."
+ local snapshot_timestamp=$(get_snapshot_timestamp "${file}")
+
+ if [ ${ignore_timestamp} == 0 ]; then
+ if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then
+ echo "Warning: Portage is newer than snapshot"
+ have_files=0
+ fi
+ else
+ utc_date=$(get_utc_from_string "${date}")
+
+ #
+ # Check that this snapshot
+ # is what it claims to be...
+ #
+ if [ ${snapshot_timestamp} -lt ${seconds} ] || \
+ [ ${snapshot_timestamp} -gt $((${seconds}+ 2*86400)) ]; then
+
+ echo "Warning: Snapshot timestamp is not in acceptable period."
+ have_files=0
+ fi
fi
- done
- else
- got_md5=1
- fi
- FILE="${FILE_ORIG}"
-
- if (($got_md5 == 0 )); then
- echo " --- No md5sum present on the mirror. (Not yet available.)"
- continue
- elif [ -s "${FILE}" ]; then
- if eval "$md5_com"; then
- echo " === snapshot $FILE is correct, using it"
- sync_local
- echo
- echo " === Snapshot has been sync'd"
- echo
- exit 0
+ fi
+
+ if [ ${have_files} != 0 ]; then
+ break;
else
- rm $FILE
+ #
+ # Remove files and use
+ # a diffeernt mirror
+ #
+ rm -f "${file}" "${digest}" "${signature}"
fi
+ done
+
+ if [ ${have_files} != 0 ]; then
+ sync_local "${file}" && r=0
+ else
+ echo "Warning: ${date} snapshot was not found."
fi
- for i in $GENTOO_MIRRORS ; do
- URI="${i}/snapshots/$FILE"
- rm -f "$FILE"
- if (eval "$FETCHCOMMAND $wgetops") && [ -s "$FILE" ]; then
- if ! eval "$md5_com"; then
- echo "md5 failed on $FILE"
- rm ${FILE}
- continue
- else
- sync_local
- echo
- echo " *** Completed websync, please now perform a normal rsync if possible."
- echo " Update is current as of the of YYYYMMDD: ${year}${month}${day}"
- echo
- exit 0
- fi
+ rm -f "${file}" "${digest}" "${signature}"
+ return "${r}"
+}
+
+do_latest_snapshot() {
+ local attempts=-1
+ local r=1
+
+ echo "Fetching most recent snapshot..."
+
+ while (( ${attempts} < 40 )) ; do
+ local day
+ local month
+ local year
+ local seconds
+
+ attempts=$(( ${attempts} + 1 ))
+
+ utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts})
+
+ day=$(get_date_part ${utc_attempt} "%d")
+ month=$(get_date_part ${utc_attempt} "%m")
+ year=$(get_date_part ${utc_attempt} "%Y")
+ utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s")
+
+ if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then
+ echo "Note: Portage content is newer than available snapshots"
+ echo "use --revert option to overide."
+ r=0
+ break;
+ fi
+
+ if do_snapshot 0 "${year}${month}${day}"; then
+ r=0
+ break;
fi
+ done
+
+ return "${r}"
+}
+main() {
+ local arg
+ local do_revert=0
+ local revert_date
+
+ [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}"
+ cd "${DISTDIR}"
+
+ for arg in $*; do
+ local v=${arg#*=}
+ case ${arg} in
+ --help)
+ echo "usage: $0 [options]"
+ echo " --verbose (-v) - verbose"
+ echo " --revert=yyyymmdd - revert to snapshot"
+ exit 0
+ ;;
+ --verbose|-v)
+ do_verbose=1
+ ;;
+ --revert=*)
+ do_revert=1
+ revert_date=${v}
+ ;;
+ *)
+ echo "Error: Invalid arguments"
+ exit 1
+ ;;
+ esac
done
-done
-rm -rf portage
+ if [ ${do_revert} != 0 ]; then
+ do_snapshot 1 "${revert_date}"
+ else
+ do_latest_snapshot
+ fi
+}
-exit 1
+main $*
projects / portage.git / commitdiff