From c6b3677e64e9d6c09d187d7dc829db0d3b348c3e Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Tue, 7 Aug 2007 08:37:13 +0000 Subject: [PATCH] Merged from trunk 7571:7583 | 7572 | Make sure that portage always has all of it's RDEPENDs | | zmedico | installed first. | | 7574 | Use == for comparison of portage_node since | | zmedico | digraph.clone() makes a deepcopy of the nodes. | | 7576 | For bug #187806, use mkstemp to create | | zmedico | tmpservertimestampfile (instead of using PORTAGE_TMPDIR). | | 7578 | Fix from ferringb | | WarnerBro | | | 7579 | Merge rewrite by Alon Bar-Lev from bug #130039: - Verify | | zmedico | gpg signature of snapshot if FEATURES="webrsync-gpg" is | | | enabled. - Add a --revert=yyyymmdd option that alows a | | | specific snapshot to be used. | | 7580 | For bug #187795, remove pointless digest.disjointed and | | zmedico | digest.notadded warnings since they are always added to | | | cvs automatically anyway. | | 7582 | Document FEATURES=webrsync-gpg. Thanks to Arfrever Frehtes | | zmedico | Taifersar Arahesis in bug #130039. | | 7583 | Update emerge --help and --info docs. Thanks to Arfrever | | zmedico | Frehtes Taifersar Arahesis in bug #137483. | svn path=/main/branches/prefix/; revision=7585 --- bin/emerge-webrsync | 416 ++++++++++++++++++++++++++++++----------- bin/repoman | 12 -- man/emerge.1 | 7 +- man/make.conf.5 | 3 + pym/emerge/__init__.py | 15 +- pym/emerge/help.py | 2 +- pym/portage/util.py | 2 +- 7 files changed, 324 insertions(+), 133 deletions(-) diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index 1e3854c62..6e5a4d46c 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -4,11 +4,21 @@ # $Id$ # Author: Karl Trygve Kalleberg # Rewritten from the old, Perl-based emerge-webrsync script +# Author: Alon Bar-Lev +# Major rewrite from Karl's scripts. + +# +# gpg key import +# KEY_ID=0x7DDAD20D +# gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID +# gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust +# type portageq > /dev/null || exit $? eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \ - PORTAGE_BIN_PATH PORTAGE_INST_UID PORTAGE_INST_GID PORTAGE_NICENESS \ - PORTAGE_TMPDIR PORTDIR PORTAGE_RSYNC_EXTRA_OPTS http_proxy ftp_proxy EPREFIX) + PORTAGE_BIN_PATH PORTAGE_GPG_DIR PORTAGE_INST_UID PORTAGE_INST_GID \ + PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \ + http_proxy ftp_proxy EPREFIX) DISTDIR="${PORTAGE_TMPDIR}/emerge-webrsync" export http_proxy ftp_proxy @@ -20,146 +30,326 @@ fi source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 -if [ ! -d $DISTDIR ] ; then - mkdir -p $DISTDIR -fi +# ${USERLAND} is unreliable since the portage tree might be empty, so test +# success of the -r option to distinguish between gnu and bsd date. +date -r 0 >&/dev/null && DATE_ARGS="BSD" || DATE_ARGS="GNU" -cd "$DISTDIR" - -found=0 -if [ "$1" == "-v" ] ; then - wgetops= -else - #this sucks. probably better to do 1> /dev/null - #that said, waiting on the refactoring. - if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then - wgetops="-q" - elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then - wgetops="-s -f" - fi -fi +do_verbose=0 -if type -P md5sum > /dev/null; then - md5_com='md5sum -c "${FILE}.md5sum"' -elif type -P md5 > /dev/null; then - md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \ -f 1 ${FILE}.md5sum)" ]' +if hasq webrsync-gpg ${FEATURES} ; then + WEBSYNC_VERIFY_SIGNATURE=1 else - echo "warning, unable to do md5 verification of the snapshot!" - echo "no suitable md5/md5sum binary was found!" - md5_com='true' + WEBSYNC_VERIFY_SIGNATURE=0 fi +if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 -a -z "${PORTAGE_GPG_DIR}" ]; then + echo "Error: Please set PORTAGE_GPG_DIR in make.conf" + exit 1 +fi + +get_utc_date_in_seconds() { + date -u +"%s" +} + +get_date_part() { + local utc_time_in_secs="$1" + local part="$2" + + if [ "${DATE_ARGS}" = "BSD" ]; then + date -r ${utc_time_in_secs} -u +"${part}" + else + date -d @${utc_time_in_secs} -u +"${part}" + fi +} + +get_utc_from_string() { + local s="$1" + + seconds=$(date -d "${s:0:4}-${s:4:2}-${s:6:2}" -u +"%s") +} + +get_portage_timestamp() { + local portage_current_timestamp=0 + + if [ -f "${PORTDIR}/metadata/timestamp.x" ]; then + portage_current_timestamp=$(cut -f 1 -d " " "${PORTDIR}/metadata/timestamp.x" ) + fi + + echo "${portage_current_timestamp}" +} + +fetch_file() { + local URI="$1" + local FILE="$2" + local opts + + if [ "${FETCHCOMMAND/wget/}" != "${FETCHCOMMAND}" ]; then + opts="--continue" + + [ "${do_verbose}" == 0 ] && opts="$opts -q" + elif [ "${FETCHCOMMAND/curl/}" != "${FETCHCOMMAND}" ]; then + opts="--continue-at -" + + [ "${do_verbose}" == 0 ] && opts="$opts -s -f" + else + rm -f "${FILE}" + fi + + echo "Fetching file ${FILE}..." + + #already set DISTDIR= + if [ "${do_verbose}" == 0 ] ; then + eval "${FETCHCOMMAND}" ${opts} > /dev/null && [ -s "${FILE}" ] + else + eval "${FETCHCOMMAND}" ${opts} && [ -s "${FILE}" ] + fi +} + +check_file_digest() { + local digest="$1" + local file="$2" + local r=1 + + echo "Checking digest..." + + if type -P md5sum > /dev/null; then + md5sum -c $digest && r=0 + elif type -P md5 > /dev/null; then + [ "$(md5 -q $file)" == "$(cut -d \ -f 1 \"$digest\")" ] && r=0 + else + echo "Error: Cannot check digest" + echo "No suitable md5/md5sum binaries found" + fi + + return "${r}" +} + +check_file_signature() { + local signature="$1" + local file="$2" + local r=1 + + if [ ${WEBSYNC_VERIFY_SIGNATURE} != 0 ]; then + + echo "Checking signature..." + + if type -p gpg > /dev/null; then + gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0 + else + echo "Error: Cannot check signature" + echo "gpg binary not found" + fi + else + r=0 + fi + + return "${r}" +} + +get_snapshot_timestamp() { + local file="$1" + + tar --to-stdout -xf "${file}" portage/metadata/timestamp.x | cut -f 1 -d " " +} sync_local() { - echo Syncing local tree... + local file="$1" + + echo "Syncing local tree..." + if type -P tarsync &> /dev/null; then - # tarsync doesn't take numeric uid/gid so we need to convert them. - local inst_user="$(python -c "import pwd; print pwd.getpwuid(int('${PORTAGE_INST_UID:-0}'))[0]")" - local inst_group="$(python -c "import grp; print grp.getgrgid(int('${PORTAGE_INST_GID:-0}'))[0]")" - if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o ${inst_user} -g ${inst_group} -e /distfiles -e /packages -e /local; then - echo "tarsync failed; tarball is corrupt?" - exit 1; + if [ "${do_verbose}" != 0 ] ; then + tarsync_verbose=-v + fi + if ! tarsync $tarsync_verbose -s 1 -o portage -g portage -e /distfiles -e /packages -e /local "${file}" "${PORTDIR}"; then + echo "Error: tarsync failed; tarball is corrupt? (${file})" + return 1 fi - rm "${FILE}" else - if ! tar jxf $FILE; then - echo "Tar failed to extract the image. Please review the output." - echo "Executed command: tar jxf $FILE" - exit 1 + echo "Note: tarsync was not found, you may consider emerge it..." + + if ! tar jxf "${file}"; then + echo "Error: tar failed to extract the image. tarball is corrupt? (${file})" + rm -fr portage + return 1 fi - rm -f $FILE + + # Free disk space + rm -f "${file}" + # Make sure user and group file ownership is ${PORTAGE_INST_UID}:${PORTAGE_INST_GID} chown -R ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} portage cd portage rsync -av --progress --stats --delete --delete-after \ - --exclude='/distfiles' --exclude='/packages' \ - --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . ${PORTDIR%%/} + --exclude='/distfiles' --exclude='/packages' \ + --exclude='/local' ${PORTAGE_RSYNC_EXTRA_OPTS} . "${PORTDIR%%/}" cd .. - echo "cleaning up" - rm -rf portage + + echo "Cleaning up..." + rm -fr portage fi + if hasq metadata-transfer ${FEATURES} ; then - echo "transferring metadata/cache" + echo "Updating cache..." emerge --metadata fi [ -x "${EPREFIX}"/etc/portage/bin/post_sync ] && "${EPREFIX}"/etc/portage/bin/post_sync } -echo "Fetching most recent snapshot" - -declare -i attempts=0 -while (( $attempts < 40 )) ; do - attempts=$(( attempts + 1 )) - - # The snapshot for a given day is generated at 01:45 UTC on the following - # day, so the current day's snapshot (going by UTC time) hasn't been - # generated yet. Therefore, always start by looking for the previous day's - # snapshot (for attempts=1, subtract 1 day from the current UTC time). - daysbefore=$(expr $(date -u +"%s") - 86400 \* ${attempts}) - DATE_ARGS="-d @${daysbefore}" - # ${USERLAND} is unreliable since the portage tree might be empty, so test - # success of the -r option to distinguish between gnu and bsd date. - date -r ${daysbefore} >&/dev/null && DATE_ARGS="-r ${daysbefore}" - day=$(date ${DATE_ARGS} -u +"%d") - month=$(date ${DATE_ARGS} -u +"%m") - year=$(date ${DATE_ARGS} -u +"%Y") - - FILE_ORIG="portage-${year}${month}${day}.tar.bz2" - - echo "Attempting to fetch file dated: ${year}${month}${day}" - - got_md5=0 - - if [ ! -e "${FILE_ORIG}.md5sum" ]; then - FILE="${FILE_ORIG}.md5sum" - for i in $GENTOO_MIRRORS ; do - URI="${i}/snapshots/${FILE}" - if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then - got_md5=1 - break +do_snapshot() { + local ignore_timestamp="$1" + local date="$2" + + local r=1 + + local file="portage-${date}.tar.bz2" + local digest="${file}.md5sum" + local signature="${file}.gpgsig" + + local have_files=0 + local mirror + + echo "Trying to retrieve ${date} snapshot..." + + for mirror in ${GENTOO_MIRRORS} ; do + + [ -s "${file}" -a -s "${digest}" -a -s "${signature}" ] && \ + check_file_digest "${digest}" "${file}" && \ + check_file_signature "${signature}" "${file}" && \ + have_files=1 + + [ ${have_files} == 0 ] && \ + fetch_file "${mirror}/snapshots/${digest}" "${digest}" && \ + fetch_file "${mirror}/snapshots/${signature}" "${signature}" && \ + fetch_file "${mirror}/snapshots/${file}" "${file}" && \ + check_file_digest "${digest}" "${file}" && \ + check_file_signature "${signature}" "${file}" && \ + have_files=1 + + # + # If timestamp is invalid + # we want to try and retieve + # from a different mirror + # + if [ ${have_files} != 0 ]; then + + echo "Getting snapshot timetasmp..." + local snapshot_timestamp=$(get_snapshot_timestamp "${file}") + + if [ ${ignore_timestamp} == 0 ]; then + if [ ${snapshot_timestamp} -lt $(get_portage_timestamp) ]; then + echo "Warning: Portage is newer than snapshot" + have_files=0 + fi + else + utc_date=$(get_utc_from_string "${date}") + + # + # Check that this snapshot + # is what it claims to be... + # + if [ ${snapshot_timestamp} -lt ${seconds} ] || \ + [ ${snapshot_timestamp} -gt $((${seconds}+ 2*86400)) ]; then + + echo "Warning: Snapshot timestamp is not in acceptable period." + have_files=0 + fi fi - done - else - got_md5=1 - fi - FILE="${FILE_ORIG}" - - if (($got_md5 == 0 )); then - echo " --- No md5sum present on the mirror. (Not yet available.)" - continue - elif [ -s "${FILE}" ]; then - if eval "$md5_com"; then - echo " === snapshot $FILE is correct, using it" - sync_local - echo - echo " === Snapshot has been sync'd" - echo - exit 0 + fi + + if [ ${have_files} != 0 ]; then + break; else - rm $FILE + # + # Remove files and use + # a diffeernt mirror + # + rm -f "${file}" "${digest}" "${signature}" fi + done + + if [ ${have_files} != 0 ]; then + sync_local "${file}" && r=0 + else + echo "Warning: ${date} snapshot was not found." fi - for i in $GENTOO_MIRRORS ; do - URI="${i}/snapshots/$FILE" - rm -f "$FILE" - if (eval "$FETCHCOMMAND $wgetops") && [ -s "$FILE" ]; then - if ! eval "$md5_com"; then - echo "md5 failed on $FILE" - rm ${FILE} - continue - else - sync_local - echo - echo " *** Completed websync, please now perform a normal rsync if possible." - echo " Update is current as of the of YYYYMMDD: ${year}${month}${day}" - echo - exit 0 - fi + rm -f "${file}" "${digest}" "${signature}" + return "${r}" +} + +do_latest_snapshot() { + local attempts=-1 + local r=1 + + echo "Fetching most recent snapshot..." + + while (( ${attempts} < 40 )) ; do + local day + local month + local year + local seconds + + attempts=$(( ${attempts} + 1 )) + + utc_attempt=$(expr $(get_utc_date_in_seconds) - 86400 \* ${attempts}) + + day=$(get_date_part ${utc_attempt} "%d") + month=$(get_date_part ${utc_attempt} "%m") + year=$(get_date_part ${utc_attempt} "%Y") + utc_midnight=$(get_date_part $(expr ${utc_attempt} - ${utc_attempt} % 86400) "%s") + + if [ ${utc_midnight} -lt $(($(get_portage_timestamp)-86400)) ]; then + echo "Note: Portage content is newer than available snapshots" + echo "use --revert option to overide." + r=0 + break; + fi + + if do_snapshot 0 "${year}${month}${day}"; then + r=0 + break; fi + done + + return "${r}" +} +main() { + local arg + local do_revert=0 + local revert_date + + [ ! -d "${DISTDIR}" ] && mkdir -p "${DISTDIR}" + cd "${DISTDIR}" + + for arg in $*; do + local v=${arg#*=} + case ${arg} in + --help) + echo "usage: $0 [options]" + echo " --verbose (-v) - verbose" + echo " --revert=yyyymmdd - revert to snapshot" + exit 0 + ;; + --verbose|-v) + do_verbose=1 + ;; + --revert=*) + do_revert=1 + revert_date=${v} + ;; + *) + echo "Error: Invalid arguments" + exit 1 + ;; + esac done -done -rm -rf portage + if [ ${do_revert} != 0 ]; then + do_snapshot 1 "${revert_date}" + else + do_latest_snapshot + fi +} -exit 1 +main $* diff --git a/bin/repoman b/bin/repoman index 4f00f95d0..e3d0fc4b3 100755 --- a/bin/repoman +++ b/bin/repoman @@ -137,8 +137,6 @@ qahelp={ "digest.fail":"Digest does not match the specified local file", "digest.stray":"Digest files that do not have a corresponding ebuild", "digest.missing":"Digest files that are missing (ebuild exists, digest doesn't)", - "digest.disjointed":"Digests not added to cvs when the matching ebuild has been added", - "digest.notadded":"Digests that exist but have not been added to cvs", "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)", "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)", "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory", @@ -209,8 +207,6 @@ qawarnings=[ "ebuild.allmasked", "ebuild.nesteddie", "digest.assumed", -"digest.notadded", -"digest.disjointed", "digest.missing", "digestentry.unused", "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked", @@ -1039,14 +1035,6 @@ for x in scanlist: for y in filesdirlist: if not y.startswith("digest-"): continue - if y[7:] not in dadded: - #digest not added to cvs or svn - stats["digest.notadded"]=stats["digest.notadded"]+1 - fails["digest.notadded"].append(x+"/files/"+y) - if y[7:] in eadded: - stats["digest.disjointed"]=stats["digest.disjointed"]+1 - fails["digest.disjointed"].append(x+"/files/"+y) - if os.stat(checkdir+"/files/"+y)[0] & 0x0248: stats["file.executable"] += 1 fails["file.executable"].append(x+"/files/"+y) diff --git a/man/emerge.1 b/man/emerge.1 index e2e37b25c..55cafe066 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -7,13 +7,16 @@ emerge \- Command\-line interface to the Portage system [\fIoptions\fR] [\fIaction\fR] [\fIebuild\fR | \fItbz2file\fR | \fIset\fR | \fIatom\fR] ... .TP .BR emerge -\fB\-\-sync\fR | \fB\-\-info\fR | \fB\-\-version\fR +\fB\-\-sync\fR | \fB\-\-version\fR +.TP +.BR emerge +\fB\-\-info\fR [\fIatom\fR] .TP .BR emerge \fB\-\-search\fR \fIsomestring\fR .TP .BR emerge -\fB\-\-help\fR [\fB\-\-system\fR | \fB\-\-config\fR | \fB\-\-sync\fR] +\fB\-\-help\fR [\fBsystem\fR | \fBworld\fR | \fB\-\-sync\fR] .SH "DESCRIPTION" \fBemerge\fR is the definitive command\-line interface to the Portage system. It is primarily used for installing packages, and \fBemerge\fR diff --git a/man/make.conf.5 b/man/make.conf.5 index 9402f2114..0071a480f 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -291,6 +291,9 @@ portage:portage without a sandbox (unless \fIusersandbox\fR is also used). .TP .B usersandbox Enable the sandbox in the compile phase, when running without root privs (\fIuserpriv\fR). +.TP +.B webrsync-gpg +Enable GPG verification when using \fIemerge\-webrsync\fR. .RE .TP .B FETCHCOMMAND diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 8b8938b92..534b93463 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -2195,12 +2195,14 @@ class depgraph(object): circular_blocks = False blocker_deps = None asap_nodes = [] + portage_node = None if reversed: get_nodes = mygraph.root_nodes else: get_nodes = mygraph.leaf_nodes for cpv, node in self.pkg_node_map["/"].iteritems(): if "portage" == portage.catsplit(portage.dep_getkey(cpv))[-1]: + portage_node = node asap_nodes.append(node) break ignore_priority_soft_range = [None] @@ -2261,6 +2263,11 @@ class depgraph(object): return True if node not in mergeable_nodes: return False + if node == portage_node and mygraph.child_nodes(node, + ignore_priority=DepPriority.MEDIUM_SOFT): + # Make sure that portage always has all of it's + # RDEPENDs installed first. + return False selected_nodes.add(node) for child in mygraph.child_nodes(node, ignore_priority=ignore_priority): @@ -4474,7 +4481,6 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if myaction == "metadata": print "skipping sync" updatecache_flg = True - tmpservertimestampfile = None elif syncuri[:8]=="rsync://": if not os.path.exists(EPREFIX+"/usr/bin/rsync"): print "!!! rsync does not exist, so rsync support is disabled." @@ -4589,9 +4595,6 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # Real local timestamp file. servertimestampfile = os.path.join( myportdir, "metadata", "timestamp.chk") - # Temporary file for remote server timestamp comparison. - tmpservertimestampfile = os.path.join( - settings["PORTAGE_TMPDIR"], "timestamp.chk") content = portage.util.grabfile(servertimestampfile) mytimestamp = 0 @@ -4705,6 +4708,10 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # connection attempt to an unresponsive server which rsync's # --timeout option does not prevent. if True: + # Temporary file for remote server timestamp comparison. + from tempfile import mkstemp + fd, tmpservertimestampfile = mkstemp() + os.close(fd) mycommand = rsynccommand[:] mycommand.append(dosyncuri.rstrip("/") + \ "/metadata/timestamp.chk") diff --git a/pym/emerge/help.py b/pym/emerge/help.py index bbb17abe2..ee0ba19ee 100644 --- a/pym/emerge/help.py +++ b/pym/emerge/help.py @@ -13,7 +13,7 @@ def shorthelp(): print " "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] < "+turquoise("system")+" | "+turquoise("world")+" >" print " "+turquoise("emerge")+" < "+turquoise("--sync")+" | "+turquoise("--metadata")+" | "+turquoise("--info")+" >" print " "+turquoise("emerge")+" "+turquoise("--resume")+" [ "+green("--pretend")+" | "+green("--ask")+" | "+green("--skipfirst")+" ]" - print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("config")+" | "+green("--sync")+" ] " + print " "+turquoise("emerge")+" "+turquoise("--help")+" [ "+green("system")+" | "+green("world")+" | "+green("--sync")+" ] " print bold("Options:")+" "+green("-")+"["+green("abBcCdDefgGhikKlnNoOpqPsStuvV")+"] ["+green("--oneshot")+"] ["+green("--newuse")+"] ["+green("--noconfmem")+"]" print " [ " + green("--color")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ] [ "+green("--columns")+" ]" print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] ["+green("--nospinner")+"]" diff --git a/pym/portage/util.py b/pym/portage/util.py index 9d3dc6459..81400e3c6 100644 --- a/pym/portage/util.py +++ b/pym/portage/util.py @@ -247,7 +247,7 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0): pkgs=grabfile(myfilename, compatlevel, recursive=recursive) for x in range(len(pkgs)-1, -1, -1): pkg = pkgs[x] - if pkg[0] in ["-","*"]: # -atom *pkg for packages file + if pkg[0] in '-*': # -atom *pkg for packages file pkg = pkg[1:] if not isvalidatom(pkg): writemsg("--- Invalid atom in %s: %s\n" % (myfilename, pkgs[x]), -- 2.26.2