From 581e83296bf0f0cd43f81f087382bb2378070167 Mon Sep 17 00:00:00 2001 From: Kito Date: Mon, 20 Mar 2006 21:12:29 +0000 Subject: [PATCH] merge with trunk rev 2947:2966 svn path=/main/branches/prefix/; revision=2967 --- NEWS | 1 + RELEASE-NOTES | 2 + bin/ebuild.sh | 28 +++---- bin/emerge | 150 ++++++++++++++++++++++++++++---------- bin/emerge-webrsync | 8 +- bin/isolated-functions.sh | 17 +++++ bin/repoman | 3 +- cnf/make.conf | 56 +++----------- cnf/make.conf.alpha | 19 +++-- cnf/make.conf.amd64 | 19 +++-- cnf/make.conf.arm | 19 +++-- cnf/make.conf.hppa | 19 +++-- cnf/make.conf.ia64 | 19 +++-- cnf/make.conf.mac | 6 -- cnf/make.conf.mips | 19 +++-- cnf/make.conf.ppc | 19 +++-- cnf/make.conf.ppc64 | 19 +++-- cnf/make.conf.s390 | 19 +++-- cnf/make.conf.sparc | 19 +++-- cnf/make.conf.x86 | 19 +++-- cnf/make.conf.x86-fbsd | 19 +++-- cnf/make.globals | 6 +- man/make.conf.5 | 38 +++++----- pym/portage.py | 42 +++++------ pym/portage_util.py | 5 ++ 25 files changed, 315 insertions(+), 275 deletions(-) delete mode 100644 cnf/make.conf.mac diff --git a/NEWS b/NEWS index 559e8415c..c126c3211 100644 --- a/NEWS +++ b/NEWS @@ -34,3 +34,4 @@ portage-2.1 (ongoing via pre releases) * Global updates and fixpackages performance optimizations. * Tests show that file locking is now more reliable. * A bash call stack is printed when an ebuild dies in ebuild.sh. +* New rsync option handling by using a generic PORTAGE_RSYNC_EXTRA_OPTS variable diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 4a2747f5d..0e6753b12 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -16,3 +16,5 @@ portage-2.1 for other actions) * Tools that call emerge should override the EMERGE_DEFAULT_OPTS environment variable or use the emerge --ignore-default-opts option. +* rsync option handling has been redesigned, instead of RSYNC_* variables + use PORTAGE_RSYNC_EXTRA_OPTS from now on. diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 5b5976712..6ab87581b 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -52,8 +52,16 @@ export PATH="${DEFAULT_PATH}:${ROOTPATH}" source @PORTAGE_BASE@/bin/isolated-functions.sh &>/dev/null -# TODO: make this conditional on config settings, fix any remaining stuff -set_colors + +case "${NOCOLOR:-false}" in + yes|true) + unset_colors + ;; + no|false) + set_colors + ;; +esac + # the sandbox is disabled by default except when overridden in the relevant stages export SANDBOX_ON="0" @@ -137,22 +145,6 @@ useq() { fi } -has() { - hasq "$@" -} - -hasv() { - if hasq "$@" ; then - echo "$1" - return 0 - fi - return 1 -} - -hasq() { - [[ " ${*:2} " == *" $1 "* ]] -} - has_version() { if [ "${EBUILD_PHASE}" == "depend" ]; then die "portageq calls (has_version calls portageq) are not allowed in the global scope" diff --git a/bin/emerge b/bin/emerge index 7604dc93d..c7d8b493e 100755 --- a/bin/emerge +++ b/bin/emerge @@ -408,8 +408,14 @@ if ("--resume" in myopts): myopts.remove("--tree") # Set color output -if ("--nocolor" in myopts) and (sys.stdout.isatty()): +if "--nocolor" in myopts or \ +portage.settings["NOCOLOR"] in ("yes","true"): nocolor() + if "NOCOLOR" not in portage.settings: + portage.settings.unlock() + portage.settings["NOCOLOR"] = "true" + portage.settings.backup_changes("NOCOLOR") + portage.settings.lock() if not ("--quiet" in myopts): if not sys.stdout.isatty() or ("--nospinner" in myopts): @@ -2504,53 +2510,105 @@ if myaction in ["sync","metadata"] and (not "--help" in myopts): print "!!! Type \"emerge net-misc/rsync\" to enable rsync support." sys.exit(1) mytimeout=180 - if portage.settings.has_key("RSYNC_TIMEOUT"): - try: - mytimeout=int(portage.settings["RSYNC_TIMEOUT"]) - except SystemExit, e: - raise # Needed else can't exit - except: - pass - - rsync_flags = [ - "--recursive", # Recurse directories - "--links", # Consider symlinks - "--safe-links", # Ignore links outside of tree - "--perms", # Preserve permissions - "--times", # Preserive mod times - "--compress", # Compress the data transmitted - "--force", # Force deletion on non-empty dirs - "--whole-file", # Don't do block transfers, only entire files - "--delete", # Delete files that aren't in the master tree - "--delete-after", # Delete only after everything else is done - "--stats", # Show final statistics about what was transfered - "--timeout="+str(mytimeout), # IO timeout if not done in X seconds - "--exclude='/distfiles'", # Exclude distfiles from consideration - "--exclude='/local'", # Exclude local from consideration - "--exclude='/packages'", # Exclude packages from consideration - ] + + rsync_opts = [] + + if portage.settings["PORTAGE_RSYNC_OPTS"] == "": + portage.writemsg("PORTAGE_RSYNC_OPTS empty or unset, using hardcoded defaults\n") + rsync_opts.extend([ + "--recursive", # Recurse directories + "--links", # Consider symlinks + "--safe-links", # Ignore links outside of tree + "--perms", # Preserve permissions + "--times", # Preserive mod times + "--compress", # Compress the data transmitted + "--force", # Force deletion on non-empty dirs + "--whole-file", # Don't do block transfers, only entire files + "--delete", # Delete files that aren't in the master tree + "--delete-after", # Delete only after everything else is done + "--stats", # Show final statistics about what was transfered + "--timeout="+str(mytimeout), # IO timeout if not done in X seconds + "--exclude='/distfiles'", # Exclude distfiles from consideration + "--exclude='/local'", # Exclude local from consideration + "--exclude='/packages'", # Exclude packages from consideration + ]) + + else: + # The below validation is not needed when using the above hardcoded + # defaults. + + portage.writemsg("Using PORTAGE_RSYNC_OPTS instead of hardcoded defaults\n") + rsync_opts.extend(portage.settings["PORTAGE_RSYNC_OPTS"].split()) + + for opt in ("--recursive", "--times"): + if opt not in rsync_opts: + portage.writemsg(yellow("WARNING:") + " adding required option " + \ + "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) + rsync_opts.append(opt) + + for exclude in ("distfiles", "local", "packages"): + opt = "--exclude='/%s'" % exclude + if opt not in rsync_opts: + portage.writemsg(yellow("WARNING:") + \ + " adding required option %s not included in " % opt + \ + "PORTAGE_RSYNC_OPTS (override with --exclude='!')\n") + rsync_opts.append(opt) + + if portage.settings["RSYNC_TIMEOUT"] != "": + portage.writemsg("WARNING: usage of RSYNC_TIMEOUT is deprecated, " + \ + "use PORTAGE_RSYNC_OPTS instead\n") + try: + mytimeout = int(portage.settings["RSYNC_TIMEOUT"]) + rsync_opts.append("--timeout=%d" % mytimeout) + except ValueError, e: + portage.writemsg("!!! %s\n" % str(e)) + + # TODO: determine options required for official servers + if syncuri.rstrip("/").endswith(".gentoo.org/gentoo-portage"): + + def rsync_opt_startswith(opt_prefix): + for x in rsync_opts: + if x.startswith(opt_prefix): + return True + return False + + if not rsync_opt_startswith("--timeout="): + rsync_opts.append("--timeout=%d" % mytimeout) + + for opt in ("--compress", "--whole-file"): + if opt not in rsync_opts: + portage.writemsg(yellow("WARNING:") + " adding required option " + \ + "%s not included in PORTAGE_RSYNC_OPTS\n" % opt) + rsync_opts.append(opt) if "--quiet" in myopts: - rsync_flags.append("--quiet") # Shut up a lot + rsync_opts.append("--quiet") # Shut up a lot else: - rsync_flags.append("--progress") # Progress meter for each file + rsync_opts.append("--progress") # Progress meter for each file if "--verbose" in myopts: - rsync_flags.append("--verbose") # More noise? Not really sure what + rsync_opts.append("--verbose") # More noise? Not really sure what if "--debug" in myopts: - rsync_flags.append("--checksum") # Force checksum on all files + rsync_opts.append("--checksum") # Force checksum on all files - if portage.settings.has_key("RSYNC_EXCLUDEFROM"): + if portage.settings["RSYNC_EXCLUDEFROM"] != "": + portage.writemsg(yellow("WARNING:") + \ + " usage of RSYNC_EXCLUDEFROM is deprecated, use " + \ + "PORTAGE_RSYNC_EXTRA_OPTS instead\n") if os.path.exists(portage.settings["RSYNC_EXCLUDEFROM"]): - rsync_flags.append("--exclude-from="+portage.settings["RSYNC_EXCLUDEFROM"]) + rsync_opts.append("--exclude-from=%s" % \ + portage.settings["RSYNC_EXCLUDEFROM"]) else: - print "!!! RSYNC_EXCLUDEFROM specified, but file does not exist." + portage.writemsg("!!! RSYNC_EXCLUDEFROM specified," + \ + " but file does not exist.\n") - if portage.settings.has_key("RSYNC_RATELIMIT"): - rsync_flags.append("--bwlimit="+portage.settings["RSYNC_RATELIMIT"]) - - rsynccommand = "rsync " + string.join(rsync_flags, " ") + if portage.settings["RSYNC_RATELIMIT"] != "": + portage.writemsg(yellow("WARNING:") + \ + " usage of RSYNC_RATELIMIT is deprecated, use " + \ + "PORTAGE_RSYNC_EXTRA_OPTS instead") + rsync_opts.append("--bwlimit=%s" % \ + portage.settings["RSYNC_RATELIMIT"]) servertimestampdir = portage.settings.depcachedir+"/" servertimestampfile = portage.settings.depcachedir+"/timestamp.chk" @@ -2579,7 +2637,11 @@ if myaction in ["sync","metadata"] and (not "--help" in myopts): #exitcode=0 try: - maxretries=int(portage.settings["RSYNC_RETRIES"]) + if portage.settings.has_key("RSYNC_RETRIES"): + print yellow("WARNING:")+" usage of RSYNC_RETRIES is deprecated, use PORTAGE_RSYNC_RETRIES instead" + maxretries=int(portage.settings["RSYNC_RETRIES"]) + else: + maxretries=int(portage.settings["PORTAGE_RSYNC_RETRIES"]) except SystemExit, e: raise # Needed else can't exit except: @@ -2629,7 +2691,17 @@ if myaction in ["sync","metadata"] and (not "--help" in myopts): if "--quiet" not in myopts: print ">>> Checking server timestamp ..." - mycommand=rsynccommand+" "+dosyncuri+"/metadata/timestamp.chk "+tmpservertimestampdir + + rsynccommand = " ".join(["/usr/bin/rsync", + portage.settings["PORTAGE_RSYNC_EXTRA_OPTS"], + " ".join(rsync_opts)]) + + if "--debug" in myopts: + print rsynccommand + + mycommand = " ".join([rsynccommand, + dosyncuri + "/metadata/timestamp.chk", + tmpservertimestampdir]) exitcode=portage.spawn(mycommand,portage.settings,free=1) if (exitcode==0): try: diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index 882f66061..91c32f5f7 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -12,6 +12,8 @@ if [ -n "${PORTAGE_NICENESS}" ]; then renice $PORTAGE_NICENESS $$ > /dev/null fi +source @PORTAGE_BASE@/bin/isolated-functions.sh || exit 1 +FEATURES="$(@PORTAGE_BASE@/bin/portageq envvar FEATURES)" GENTOO_MIRRORS="${GENTOO_MIRRORS} $(@PORTAGE_BASE@/bin/portageq gentoo_mirrors)" PORTDIR="$(@PORTAGE_BASE@/bin/portageq portdir)" FETCHCOMMAND="$(@PORTAGE_BASE@/bin/portageq envvar FETCHCOMMAND)" @@ -74,8 +76,10 @@ sync_local() { echo "cleaning up" rm -rf portage fi - echo "transferring metadata/cache" - emerge --metadata + if hasq metadata-transfer ${FEATURES} ; then + echo "transferring metadata/cache" + emerge --metadata + fi } echo "Fetching most recent snapshot" diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 20be25e77..c257a10d5 100644 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -206,4 +206,21 @@ RC_ENDCOL="yes" RC_INDENTATION='' RC_DEFAULT_INDENT=2 RC_DOT_PATTERN='' + +has() { + hasq "$@" +} + +hasv() { + if hasq "$@" ; then + echo "$1" + return 0 + fi + return 1 +} + +hasq() { + [[ " ${*:2} " == *" $1 "* ]] +} + true diff --git a/bin/repoman b/bin/repoman index 9ca26e69f..d57ac4e36 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1064,7 +1064,7 @@ for x in scanlist: stats["KEYWORDS.stupid"] += 1 fails["KEYWORDS.stupid"].append(x+"/"+y+".ebuild") - if "--ignore-other-arches" in myoptions: + if "--ignore-arches" in myoptions: arches=[[repoman_settings["ARCH"], repoman_settings["ARCH"], portage.groups]] else: arches=[] @@ -1401,7 +1401,6 @@ for x in qacats: else: for y in fails[x]: print " "+y -print def grouplist(mylist,seperator="/"): """(list,seperator="/") -- Takes a list of elements; groups them into diff --git a/cnf/make.conf b/cnf/make.conf index 863fb957e..d993464ca 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -23,15 +23,6 @@ # Example: #USE="X gtk gnome -alsa" -# Host Setting -# ============ -# -# DO NOT CHANGE THIS SETTING UNLESS YOU ARE USING STAGE1! -# Change this line as appropriate (i686, i586, i486 or i386). -# All modern systems (even Athlons) should use "i686-pc-linux-gnu". -# All K6's are i586. -CHOST="i686-pc-linux-gnu" - # Host and optimization settings # ============================== # @@ -42,34 +33,10 @@ CHOST="i686-pc-linux-gnu" # package (and in some cases the libraries it uses) at default optimizations # before reporting errors to developers. # -# -mcpu= means optimize code for the particular type of CPU without -# breaking compatibility with other CPUs. -# -# -march= means to take full advantage of the ABI and instructions -# for the particular CPU; this will break compatibility with older CPUs (for -# example, -march=athlon-xp code will not run on a regular Athlon, and -# -march=i686 code will not run on a Pentium Classic. -# -# CPU types supported in gcc-3.2 and higher: athlon-xp, athlon-mp, -# athlon-tbird, athlon, k6, k6-2, k6-3, i386, i486, i586 (Pentium), i686 -# (PentiumPro), pentium, pentium-mmx, pentiumpro, pentium2 (Celeron), -# pentium3, and pentium4. -# -# Note that Gentoo Linux 1.4 and higher include at least gcc-3.2. -# -# CPU types supported in gcc-2.95*: k6, i386, i486, i586 (Pentium), i686 -# (Pentium Pro), pentium, pentiumpro Gentoo Linux 1.2 and below use gcc-2.95* -# -# CRITICAL WARNINGS: ****************************************************** # -# K6 markings are deceptive. Avoid setting -march for them. See Bug #24379. # -# Pentium-M CPU's should not enable sse2 until at least gcc-3.4. Bug 50616. # -# ************************************************************************* # +# Please refer to the GCC manual for a list of possible values. # -# Decent examples: +#CFLAGS="-O2 -pipe" # -#CFLAGS="-mcpu=athlon-xp -O3 -pipe" -#CFLAGS="-march=pentium3 -O3 -pipe" - # If you set a CFLAGS above, then this line will set your default C++ flags to # the same settings. #CXXFLAGS="${CFLAGS}" @@ -207,18 +174,17 @@ CHOST="i686-pc-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.alpha b/cnf/make.conf.alpha index 613c2abb4..6a5609d61 100644 --- a/cnf/make.conf.alpha +++ b/cnf/make.conf.alpha @@ -193,18 +193,17 @@ CFLAGS="-mcpu=ev5 -O3 -pipe " # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.amd64 b/cnf/make.conf.amd64 index ef4ef3212..2b83eb566 100644 --- a/cnf/make.conf.amd64 +++ b/cnf/make.conf.amd64 @@ -204,18 +204,17 @@ CHOST="x86_64-pc-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.arm b/cnf/make.conf.arm index ccb4e47b5..d9a4bdacb 100644 --- a/cnf/make.conf.arm +++ b/cnf/make.conf.arm @@ -201,18 +201,17 @@ CHOST="armv4l-unknown-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.hppa b/cnf/make.conf.hppa index 892fa51c4..ffe0e32db 100644 --- a/cnf/make.conf.hppa +++ b/cnf/make.conf.hppa @@ -210,18 +210,17 @@ CHOST="hppa-unknown-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.ia64 b/cnf/make.conf.ia64 index 95df49395..270111443 100644 --- a/cnf/make.conf.ia64 +++ b/cnf/make.conf.ia64 @@ -172,18 +172,17 @@ CHOST="ia64-unknown-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.mac b/cnf/make.conf.mac deleted file mode 100644 index 292d9a510..000000000 --- a/cnf/make.conf.mac +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 1999-2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Id: /var/cvsroot/gentoo-src/portage/cnf/make.conf.mac,v 1.2 2004/09/30 06:34:27 vapier Exp $ -# Contains local system settings for Portage system - -#Nothing needed here diff --git a/cnf/make.conf.mips b/cnf/make.conf.mips index 8602b00e2..16ac25b9d 100644 --- a/cnf/make.conf.mips +++ b/cnf/make.conf.mips @@ -190,18 +190,17 @@ CHOST="mips-unknown-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.ppc b/cnf/make.conf.ppc index 1b6ebcc77..5bfc411bb 100644 --- a/cnf/make.conf.ppc +++ b/cnf/make.conf.ppc @@ -230,18 +230,17 @@ CHOST="powerpc-unknown-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.ppc64 b/cnf/make.conf.ppc64 index 80c4e1fba..f0816ead8 100644 --- a/cnf/make.conf.ppc64 +++ b/cnf/make.conf.ppc64 @@ -214,18 +214,17 @@ CHOST="powerpc64-unknown-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.s390 b/cnf/make.conf.s390 index f1f612711..55f727b4f 100644 --- a/cnf/make.conf.s390 +++ b/cnf/make.conf.s390 @@ -172,18 +172,17 @@ CHOST="s390-ibm-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.sparc b/cnf/make.conf.sparc index 637bc174e..13605c53c 100644 --- a/cnf/make.conf.sparc +++ b/cnf/make.conf.sparc @@ -208,18 +208,17 @@ # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.x86 b/cnf/make.conf.x86 index f1272ccd3..c053d7256 100644 --- a/cnf/make.conf.x86 +++ b/cnf/make.conf.x86 @@ -207,18 +207,17 @@ CHOST="i686-pc-linux-gnu" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.conf.x86-fbsd b/cnf/make.conf.x86-fbsd index 8e4ba3541..53571185a 100644 --- a/cnf/make.conf.x86-fbsd +++ b/cnf/make.conf.x86-fbsd @@ -207,18 +207,17 @@ CHOST="i686-unknown-freebsd5.3" # Australia: "rsync://rsync.au.gentoo.org/gentoo-portage" #SYNC="rsync://rsync.gentoo.org/gentoo-portage" # -# RSYNC_RETRIES sets the number of times portage will attempt to retrieve +# PORTAGE_RSYNC_RETRIES sets the number of times portage will attempt to retrieve # a current portage tree before it exits with an error. This allows # for a more successful retrieval without user intervention most times. -#RSYNC_RETRIES="3" -# -# RSYNC_TIMEOUT sets the length of time rsync will wait before it times out -# on a connection. Most users will benefit from this setting as it will -# reduce the amount of 'dead air' they experience when they run across -# the occasional, unreachable mirror. Dialup users might want to set this -# value up around the 300 second mark. -#RSYNC_TIMEOUT=180 - +#PORTAGE_RSYNC_RETRIES="3" +# +# PORTAGE_RSYNC_EXTRA_OPTS can be used to feed additional options to the rsync +# command used by `emerge --sync`. This will not change the default options +# which are set by PORTAGE_RSYNC_OPTS (don't change those unless you know +# exactly what you're doing). +#PORTAGE_RSYNC_EXTRA_OPTS="" +# # Advanced Features # ================= # diff --git a/cnf/make.globals b/cnf/make.globals index f5deb0dd5..2f53e9318 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -49,10 +49,12 @@ EMERGE_WARNING_DELAY="10" AUTOCLEAN="yes" # Number of times 'emerge --sync' will run before giving up. -RSYNC_RETRIES="3" +PORTAGE_RSYNC_RETRIES="3" # Number of seconds rsync will wait before timing out. -RSYNC_TIMEOUT="180" +#RSYNC_TIMEOUT="180" + +PORTAGE_RSYNC_OPTS="--recursive --links --safe-links --perms --times --compress --force --whole-file --delete --delete-after --stats --timeout=180 --exclude='/distfiles' --exclude='/local' --exclude='/packages'" # Minimal CONFIG_PROTECT CONFIG_PROTECT="@DOMAIN_PREFIX@/etc" diff --git a/man/make.conf.5 b/man/make.conf.5 index ae0a67bf5..cf4116953 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -298,6 +298,24 @@ emerge is running at. In other words, this will not set the nice level, it will increment it. For more information about nice levels and what are acceptable ranges, see \fBnice\fR(1). .TP +\fBPORTAGE_RSYNC_EXTRA_OPTS\fR = \fI[rsync options string]\fR +Additional rsync options to be used by \fBemerge --sync\fR. +.br +Defaults to no value. +.TP +\fBPORTAGE_RSYNC_OPTS\fR = \fI[rsync options string]\fR +Default rsync options to be used by \fBemerge --sync\fR. +.br +\fBDon't change this unless you know exactly what you're doing!\fR +.br +Defaults to "--recursive --links --safe-links --perms --times --compress --force --whole-file --delete --delete-after --stats --timeout=180 --exclude='/distfiles' --exclude='/local' --exclude='/packages'" +.TP +\fBPORTAGE_RSYNC_RETRIES\fR = \fI[NUMBER]\fR +The number of times rsync should retry on failed connections before +giving up. +.br +Defaults to 3. +.TP \fBPORTAGE_TMPDIR\fR = \fI[path]\fR Defines the location of the temporary build directories. .br @@ -330,26 +348,6 @@ images. .br Defaults to /. .TP -\fBRSYNC_EXCLUDEFROM\fR = \fI"/etc/portage/rsync_excludes"\fR -This is a file that portage will pass to rsync when it updates the portage -tree. Specific chucks of the tree may be excluded from the sync process. -This may cause dependency failures if you are not careful. The file format -is one pattern per line, blanks and ';' or '#' lines are comments. See -\fBrsync\fR(1) for more details. -.TP -\fBRSYNC_RETRIES\fR = \fI[NUMBER]\fR -The number of times rsync should retry on failed connections before -giving up. -.br -Defaults to 3. -.TP -\fBRSYNC_TIMEOUT\fR = \fI[SECONDS]\fR -The number of seconds rsync should remain idle before it determines the -connection has timed out. Dialup users may need to set this value at or -above 300 seconds. -.br -Defaults to 180 seconds. -.TP \fBRPMDIR\fR = \fI[path]\fR Defines the location where created RPM packages will be stored. .br diff --git a/pym/portage.py b/pym/portage.py index 0d71ec03e..e397be41b 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -30,10 +30,9 @@ try: import commands from time import sleep from random import shuffle - from cache.cache_errors import CacheError except ImportError, e: sys.stderr.write("\n\n") - sys.stderr.write("!!! Failed to complete python imports. There are internal modules for\n") + sys.stderr.write("!!! Failed to complete python imports. These are internal modules for\n") sys.stderr.write("!!! python and failure here indicates that you have a problem with python\n") sys.stderr.write("!!! itself and thus portage is not able to continue processing.\n\n") @@ -50,6 +49,7 @@ except ImportError: bsd_chflags = None try: + from cache.cache_errors import CacheError import cvstree import xpak import getbinpkg @@ -1644,15 +1644,17 @@ def spawn(mystring,mysettings,debug=0,free=0,droppriv=0,sesandbox=0,fd_pipes=Non if free: keywords["opt_name"] += " bash" + spawn_func = portage_exec.spawn_bash else: keywords["opt_name"] += " sandbox" + spawn_func = portage_exec.spawn_sandbox if sesandbox: con = selinux.getcontext() con = string.replace(con, mysettings["PORTAGE_T"], mysettings["PORTAGE_SANDBOX_T"]) selinux.setexec(con) - retval = portage_exec.spawn_bash(mystring, env=env, **keywords) + retval = spawn_func(mystring, env=env, **keywords) if sesandbox: selinux.setexec(None) @@ -2118,24 +2120,22 @@ def digestgen(myarchives,mysettings,overwrite=1,manifestonly=0): if mydigests==None: # There was a problem, exit with an errorcode. return 0 - try: - outfile=open(digestfn, "w+") - except SystemExit, e: - raise - except Exception, e: - print "!!! Filesystem error skipping generation. (Read-Only?)" - print "!!!",e - return 0 - for x in digestCreateLines(myarchives, mydigests): - outfile.write(x+"\n") - outfile.close() - try: - os.chown(digestfn,os.getuid(),portage_gid) - os.chmod(digestfn,0664) - except SystemExit, e: - raise - except Exception,e: - print e + if mydigests != myolddigest: + digest_lines = digestCreateLines(myarchives, mydigests) + digest_success = True + try: + write_atomic(digestfn, "\n".join(digest_lines) + "\n") + digest_success = apply_secpass_permissions( + digestfn, gid=portage_gid, mode=0664) + except (IOError, OSError), e: + writemsg("!!! %s\n" % str(e)) + digest_success = False + except portage_exception.PortageException: + writemsg("!!! %s\n" % str(e)) + digest_success = False + if not digest_success: + writemsg("!!! Filesystem error, skipping generation.\n") + return 0 print green(">>> Generating the manifest file...") mypfiles=listdir(pbasedir,recursive=1,filesonly=1,ignorecvs=1,EmptyOnError=1) diff --git a/pym/portage_util.py b/pym/portage_util.py index aa84fa74e..ae5312175 100644 --- a/pym/portage_util.py +++ b/pym/portage_util.py @@ -628,6 +628,11 @@ class atomic_ofstream(file): pass except FileNotFound: pass + except OSError, oe: # from the above os.stat call + if oe.errno in (errno.ENOENT, errno.EPERM): + pass + else: + raise os.rename(self.name, self._real_name) finally: # Make sure we cleanup the temp file -- 2.26.2