From 56ce518c570a0973d4a317ba454a8c36984a153e Mon Sep 17 00:00:00 2001 From: Kito Date: Thu, 16 Feb 2006 18:24:37 +0000 Subject: [PATCH] Merge with trunk rev 2616:2718 svn path=/main/branches/prefix/; revision=2719 --- bin/doexe | 8 +- bin/dolib | 38 +++--- bin/ebuild.sh | 6 +- bin/emaint | 2 +- bin/emake | 2 +- bin/emerge | 75 ++++------- bin/quickpkg | 9 +- bin/regenworld | 4 +- bin/repoman | 6 +- man/ebuild.5 | 5 +- man/emerge.1 | 3 + man/make.conf.5 | 4 +- man/portage.5 | 2 +- man/repoman.1 | 278 ++++++++++++++++++++++++++++----------- pym/getbinpkg.py | 7 +- pym/portage.py | 258 ++++++++++++++++++------------------ pym/portage_checksum.py | 18 ++- pym/portage_data.py | 2 +- pym/portage_exception.py | 3 + pym/portage_exec.py | 36 ++++- pym/portage_locks.py | 20 ++- pym/portage_util.py | 103 +++++++++++++-- 22 files changed, 579 insertions(+), 310 deletions(-) diff --git a/bin/doexe b/bin/doexe index caae6ee3b..e14e6d107 100755 --- a/bin/doexe +++ b/bin/doexe @@ -3,12 +3,12 @@ # Distributed under the terms of the GNU General Public License v2 # $Id: /var/cvsroot/gentoo-src/portage/bin/doexe,v 1.10.2.1 2004/12/06 03:01:43 carpaski Exp $ -mynum=${#} -if [ ${mynum} -lt 1 ] ; then - echo "doexe: at least one argument needed" +if [[ $# -lt 1 ]] ; then + echo "$0: at least one argument needed" 1>&2 exit 1 fi -if [ ! -d "${D}${EXEDESTTREE}" ] ; then + +if [[ ! -d ${D}${EXEDESTTREE} ]] ; then install -d "${D}${EXEDESTTREE}" fi diff --git a/bin/dolib b/bin/dolib index 56b33fa1a..38a53ae10 100755 --- a/bin/dolib +++ b/bin/dolib @@ -3,39 +3,39 @@ # Distributed under the terms of the GNU General Public License v2 # $Id: /var/cvsroot/gentoo-src/portage/bin/dolib,v 1.8.2.2 2005/01/12 02:07:15 carpaski Exp $ +# Setup ABI cruft LIBDIR_VAR="LIBDIR_${ABI}" -if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then - CONF_LIBDIR="${!LIBDIR_VAR}" +if [[ -n ${ABI} && -n ${!LIBDIR_VAR} ]] ; then + CONF_LIBDIR=${!LIBDIR_VAR} fi unset LIBDIR_VAR - -if [ -z "${CONF_LIBDIR}" ]; then - # we need this to default to lib so that things dont break - CONF_LIBDIR="lib" -fi +# we need this to default to lib so that things dont break +CONF_LIBDIR=${CONF_LIBDIR:-lib} libdir="${D}${DESTTREE}/${CONF_LIBDIR}" -for X in 1 2 3; do - # The escaping is weird. It will break if you escape the last one. - libdir="${libdir//\/\///}" -done -if [ ${#} -lt 1 ] ; then - echo "${0}: at least one argument needed" +if [[ $# -lt 1 ]] ; then + echo "$0: at least one argument needed" 1>&2 exit 1 fi -if [ ! -d "${libdir}" ] ; then - install -d "${libdir}" +if [[ ! -d ${libdir} ]] ; then + install -d "${libdir}" || exit 1 fi +ret=0 + for x in "$@" ; do - if [ -e "${x}" ] ; then - if [ ! -L "${x}" ] ; then + if [[ -e ${x} ]] ; then + if [[ ! -L ${x} ]] ; then install ${LIBOPTIONS} "${x}" "${libdir}" else - ln -s "$(readlink "${x}")" "${libdir}/${x}" + ln -s "$(readlink "${x}")" "${libdir}/${x##*/}" fi else - echo "${0}: ${x} does not exist" + echo "!!! ${0##*/}: ${x} does not exist" 1>&2 + false fi + ((ret+=$?)) done + +exit ${ret} diff --git a/bin/ebuild.sh b/bin/ebuild.sh index bf6b9c3f4..5480dbb9a 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -419,7 +419,7 @@ diefunc() { exit 1 } -shopt -s extdebug +shopt -s extdebug &> /dev/null # usage- first arg is the number of funcs on the stack to ignore. # defaults to 1 (ignoring dump_trace) @@ -1398,7 +1398,7 @@ dyn_install() { if hasq multilib-strict ${FEATURES} && [ -x file -a -x find -a \ -n "${MULTILIB_STRICT_DIRS}" -a -n "${MULTILIB_STRICT_DENY}" ]; then - MULTILIB_STRICT_EXEMPT=$(echo ${MULTILIB_STRICT_EXEMPT:-"(perl5|gcc|gcc-lib|debug)"} | sed -e 's:\([(|)]\):\\\1:g') + MULTILIB_STRICT_EXEMPT=$(echo ${MULTILIB_STRICT_EXEMPT:-"(perl5|gcc|gcc-lib|debug|portage)"} | sed -e 's:\([(|)]\):\\\1:g') for dir in ${MULTILIB_STRICT_DIRS}; do [ -d "${D}/${dir}" ] || continue for file in $(find ${D}/${dir} -type f | grep -v "^${D}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do @@ -1931,7 +1931,7 @@ export S=${WORKDIR}/${P} unset E_IUSE E_DEPEND E_RDEPEND E_PDEPEND -for x in T P PN PV PVR PR A EBUILD EMERGE_FROM O PPID FILESDIR PORTAGE_TMPDIR; do +for x in T P PN PV PVR PR CATEGORY A EBUILD EMERGE_FROM O PPID FILESDIR PORTAGE_TMPDIR; do [[ ${!x-UNSET_VAR} != UNSET_VAR ]] && declare -r ${x} done # Need to be able to change D in dyn_preinst due to the IMAGE stuff diff --git a/bin/emaint b/bin/emaint index d9dcf6c2b..c56912a99 100755 --- a/bin/emaint +++ b/bin/emaint @@ -42,7 +42,7 @@ class WorldHandler(object): def fix(self): errors = [] try: - open(portage_const.WORLD_FILE, "w").write("\n".join(self.okay)) + portage.write_atomic(portage_const.WORLD_FILE,"\n".join(self.okay)) except OSError: errors.append(portage_const.WORLD_FILE + " could not be opened for writing") return errors diff --git a/bin/emake b/bin/emake index bfbe87aa8..5cc7deee3 100755 --- a/bin/emake +++ b/bin/emake @@ -11,4 +11,4 @@ # /etc/make.globals here because emake is only called from an # ebuild. -${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE} "$@" +exec ${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE} "$@" diff --git a/bin/emerge b/bin/emerge index 4c16742a3..66fd56c9e 100755 --- a/bin/emerge +++ b/bin/emerge @@ -9,7 +9,7 @@ sys.path = ["@PORTAGE_BASE@/pym"]+sys.path import portage -import emergehelp,xpak,string,re,commands,time,shutil,traceback,atexit,signal,socket,types +import emergehelp,xpak,string,re,commands,time,shutil,traceback,signal,socket,types from stat import * from output import * @@ -182,7 +182,8 @@ options=[ "--emptytree", "--fetchonly", "--fetch-all-uri", "--getbinpkg", "--getbinpkgonly", -"--help", "--noconfmem", +"--help", "--ignore-default-opts", +"--noconfmem", "--newuse", "--nocolor", "--nodeps", "--noreplace", "--nospinner", "--oneshot", @@ -226,7 +227,8 @@ edebug=0 # process short actions tmpcmdline=sys.argv[1:] -tmpcmdline.extend(portage.settings["EMERGE_DEFAULT_OPTS"].split()) +if "--ignore-default-opts" not in tmpcmdline: + tmpcmdline.extend(portage.settings["EMERGE_DEFAULT_OPTS"].split()) cmdline=[] for x in tmpcmdline: if x[0:1]=="-" and x[1:2]!="-": @@ -493,7 +495,7 @@ def emergeexit(): emergelog(" *** terminating.") if "notitles" not in portage.features: xtermTitleReset() -atexit.register(emergeexit) +portage.atexit_register(emergeexit) def emergeexitsig(signum, frame): signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -1495,17 +1497,17 @@ class depgraph: return myk - def display(self,mylist): + def display(self,mylist,verbosity=("--quiet" in myopts and 1 or "--verbose" in myopts and 3 or 2)): changelogs=[] p=[] totalsize=0 - if "--quiet" in myopts: + if verbosity == 1: def create_use_string(*args): return "" else: def create_use_string(name, cur_iuse, cur_use, old_iuse, old_use, is_new, - all_flags=("--verbose" in myopts), alphabetical=("--alphabetical" in myopts)): + all_flags=(verbosity == 3), alphabetical=("--alphabetical" in myopts)): enabled = [] if alphabetical: disabled = enabled @@ -1542,7 +1544,7 @@ class depgraph: ret = '%s="%s" ' % (name, ret) return ret - if "--verbose" in myopts: + if verbosity == 3: overlays = string.split(portage.settings['PORTDIR_OVERLAY']) if "--tree" in myopts: @@ -1695,7 +1697,7 @@ class depgraph: verboseadd += create_use_string(key.upper(), cur_iuse_map[key], cur_use_map[key], old_iuse_map[key], old_use_map[key], is_new) - if "--verbose" in myopts: + if verbosity == 3: # size verbose mysize=0 if x[0] == "ebuild" and x[-1]!="nomerge": @@ -1786,9 +1788,7 @@ class depgraph: ("livecvsportage" not in portage.settings.features): if mylist.index(x)>> Recording",myfavkey,"in \"world\" favorites file..." if not "--fetchonly" in myopts: - portage.writedict(myfavdict,portage.root+portage.WORLD_FILE,writekey=0) + portage.write_atomic(os.path.join(portage.root,portage.WORLD_FILE),"\n".join(myfavdict.values())) portage.mtimedb["resume"]["mergelist"]=mymergelist[:] @@ -2092,7 +2092,7 @@ class depgraph: myfavdict[myfavkey]=myfavkey print ">>> Recording",myfavkey,"in \"world\" favorites file..." emergelog(" === ("+str(mergecount)+" of "+str(len(mymergelist))+") Updating world file ("+x[pkgindex]+")") - portage.writedict(myfavdict,myroot+portage.WORLD_FILE,writekey=0) + portage.write_atomic(os.path.join(myroot,portage.WORLD_FILE),"\n".join(myfavdict.values())) if ("noclean" not in portage.features) and (x[0] != "binary"): short_msg = "emerge: ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[pkgindex]+" Clean Post" @@ -2122,43 +2122,26 @@ class depgraph: if (myver != portage.VERSION) and \ ("livecvsportage" not in portage.settings.features): if len(mymergelist) > mergecount: - myargv=sys.argv - myr=0 - for myra in range(len(myargv)): - if myargv[myr][0:len("portage")]=="portage": - del myargv[myr] - myr-=1 - if myargv[myr][0:len("sys-apps/portage")]=="sys-apps/portage": - del myargv[myr] - myr-=1 - myr+=1 emergelog(" *** RESTARTING emerge via exec() after change of portage version.") - portage.portageexit() - # Remove --ask from options before restarting - mynewargv=[] - badlongopts = ["--ask","--tree","--changelog"] - badshortopts = ["a","t","l"] - for arg in myargv: - if arg[0:2] == "--": - if arg in badlongopts: - continue - mynewargv += [arg] - elif arg[0] == "-": - myarg = "-" - for ch in arg[1:]: - if ch in badshortopts: - continue - myarg += ch - mynewargv += [myarg] - else: - mynewargv += [arg] - os.execv(portage_const.PORTAGE_BIN_PATH+"/emerge", mynewargv) + del portage.mtimedb["resume"]["mergelist"][0] + portage.run_exitfuncs() + mynewargv=[sys.argv[0],"--resume"] + badlongopts = ("--ask","--tree","--changelog","--skipfirst","--resume") + for arg in myopts: + if arg in badlongopts: + continue + mynewargv.append(arg) + os.execv(mynewargv[0], mynewargv) if ("--pretend" not in myopts) and ("--fetchonly" not in myopts) and ("--fetch-all-uri" not in myopts): emergelog(" ::: completed emerge ("+str(mergecount)+" of "+str(len(mymergelist))+") "+x[2]+" to "+x[1]) # Unsafe for parallel merges del portage.mtimedb["resume"]["mergelist"][0] + # Commit after each merge so that --resume may still work in + # in the event that portage is not allowed to exit normally + # due to power failure, SIGKILL, etc... + portage.commit_mtimedb() emergelog(" *** Finished. Cleaning up...") @@ -2756,7 +2739,7 @@ if myaction in ["sync","rsync","metadata"] and (not "--help" in myopts): sys.exit(exitcode) elif syncuri[:6]=="cvs://": if not os.path.exists("cvs"): - print "!!! cvs does not exist, so rsync support is disabled." + print "!!! cvs does not exist, so CVS support is disabled." print "!!! Type \"emerge dev-util/cvs\" to enable CVS support." sys.exit(1) cvsroot=syncuri[6:] diff --git a/bin/quickpkg b/bin/quickpkg index 61b9cd7ba..3ff5702af 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -15,6 +15,9 @@ if [ "`whoami`" != "root" ] ; then fi export PORTAGE_DB=$(portageq vdb_path) +export ROOT=$(portageq envvar ROOT) +export ROOT=${ROOT%/}/ + if [ -z "$1" ] || [ $1 == "-h" ] || [ $1 == "--help" ] ; then echo "QUICKPKG ver 1.2" echo "USAGE: quickpkg " @@ -67,8 +70,8 @@ do_pkg() { NF=NF-3 print } - }' "${SRCDIR}"/CONTENTS | cut -f2- -d" " - > "${MYDIR}"/filelist - tar vjcf "${MYDIR}"/bin.tar.bz2 --files-from="${MYDIR}"/filelist --no-recursion + }' "${SRCDIR}"/CONTENTS | cut -f2- -d" " - | sed -e 's:^/:./:' > "${MYDIR}"/filelist + tar vjcf "${MYDIR}"/bin.tar.bz2 -C "${ROOT}" --files-from="${MYDIR}"/filelist --no-recursion # join together the basic tbz2 and the pkg info files xpak "${MYDIR}"/temp "${MYDIR}"/inf.xpak @@ -109,7 +112,7 @@ for x in "$@" ; do # lets figure out what they want else - DIRLIST=$(portageq match / "${x}") + DIRLIST=$(portageq match "${ROOT}" "${x}") if [ -z "${DIRLIST}" ] ; then eerror "Could not find anything to match '${x}'; skipping" export PKGERROR="${PKGERROR} ${x}" diff --git a/bin/regenworld b/bin/regenworld index 94acf10a1..3d57d0f12 100755 --- a/bin/regenworld +++ b/bin/regenworld @@ -88,6 +88,4 @@ for mykey in biglist: print "add to world:",myfavkey worldlist.append(myfavkey) -myfile=open(portage.WORLD_FILE, "w") -myfile.write(string.join(worldlist, '\n')+'\n') -myfile.close() +portage.write_atomic(portage.WORLD_FILE,"\n".join(worldlist)) diff --git a/bin/repoman b/bin/repoman index 50c6f8b79..7d15cb3f4 100755 --- a/bin/repoman +++ b/bin/repoman @@ -39,6 +39,8 @@ from fileinput import input from grp import getgrnam from stat import * +if portage.settings["NOCOLOR"] in ("yes","true") or not sys.stdout.isatty(): + nocolor() def warn(txt): print exename+": "+txt @@ -106,7 +108,7 @@ qahelp={ "digest.disjointed":"Digests not added to cvs when the matching ebuild has been added", "digest.notadded":"Digests that exist but have not been added to cvs", "digest.unmatch":"Digests which are incomplete (please check if your USE/ARCH includes all files)", - "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name", + "ebuild.invalidname":"Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions)", "ebuild.namenomatch":"Ebuild files that do not have the same name as their parent directory", "changelog.missing":"Missing ChangeLog files", "ebuild.disjointed":"Ebuilds not added to cvs when the matching digest has been added", @@ -1751,7 +1753,7 @@ else: else: mymsg=open(commitmessagefile,"w") mymsg.write(commitmessage) - mymsg.write("\n (Unsigned Manifest commit)") + mymsg.write(" (Unsigned Manifest commit)") mymsg.close() if isCvs: retval=os.system("cvs -q commit -F "+commitmessagefile) diff --git a/man/ebuild.5 b/man/ebuild.5 index dceea7f53..ff27939b2 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -87,6 +87,9 @@ Contains the full package name \fI[PN]\-[PVR]\fR .br \fBxfree-4.2.1-r2.ebuild\fR --> \fB$PF\fR=='\fIxfree-4.2.1-r2\fR' .TP +.B CATEGORY +Contains the package category name. +.TP .B A Contains all source files required for the package. This variable must not be defined. It is autogenerated from the \fISRC_URI\fR variables. @@ -309,7 +312,7 @@ This should be a space delimited list of portage features to restrict. .RS .TP .I fetch -like \fInomirror\fR but the files will not be fetched via \fBSRC_URI\fR either. +like \fImirror\fR but the files will not be fetched via \fBSRC_URI\fR either. .TP .I mirror files in \fBSRC_URI\fR will not be downloaded from the \fBGENTOO_MIRRORS\fR. diff --git a/man/emerge.1 b/man/emerge.1 index 97532c59e..a9050f79a 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -275,6 +275,9 @@ information from the local machine. All binaries will be downloaded from the remote server without consulting packages existing in the local packages directory. .TP +.BR "\-\-ignore-default-opts" +Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored. +.TP .BR "\-\-newuse " (\fB\-N\fR) Tells emerge to include installed packages where USE flags have changed since compilation. An asterisk marks when a USE flag has changed since the package diff --git a/man/make.conf.5 b/man/make.conf.5 index fdcbd2d8c..fca815ccd 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -99,7 +99,9 @@ Defines whether or not to ignore audible beeps when displaying important informational messages. This variable is unset by default. .TP .B EMERGE_DEFAULT_OPTS -Options to append to the end of the emerge command line on every invocation. +Options to append to the end of the \fBemerge\fR(1) command line on every invocation. +These options will not be appended to the command line if --ignore-default-opts +is specified. .TP .B EPAUSE_IGNORE Defines whether or not to ignore short pauses that occur when displaying diff --git a/man/portage.5 b/man/portage.5 index b53532445..51e6f32d8 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -388,7 +388,7 @@ hosts here. If the mirror set is not found here, it will check the global mirrors file at /usr/portage/profiles/thirdpartymirrors. You may also set a special mirror type called "local". This list of mirrors will be checked before GENTOO_MIRRORS and will be used even if the package has -RESTRICT="nomirror". +RESTRICT="mirror". .I Format: .nf diff --git a/man/repoman.1 b/man/repoman.1 index 7a0deaf7c..66994edf9 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -11,120 +11,246 @@ is meant for Gentoo developers only due to the fact it does not work on a per-eb It checks the quality of CVS repositories. Note: \fBrepoman commit\fR only works \fIinside local\fR CVS repositories. -.SH OPTIONS +.SH OPTIONS .TP -.B --pretend -Don't actually perform commit or fix problems, just show what would be done. +\fB-q\fR, \fB--quiet\fR +Be less verbose about extraneous info .TP -.B --help -Show the help screen. -.SH MODES +\fB-p\fR, \fB--pretend\fR +Don't commit or fix anything; just show what would be done .TP -.B scan -Scan current directory tree for QA issues (default). See \fBQA KEYWORDS\fR for more -information. +\fB-i\fR, \fB--include-masked\fR +Include masked packages in scans at category or tree level .TP -.B fix -Fix those issues that can be fixed (stray digests, missing digests). +\fB-x\fR, \fB--xmlparse\fR +Forces the metadata.xml parse check to be carried out +.TP +\fB-v\fR, \fB--verbose\fR +Displays every package name while checking +.TP +\fB-I\fR, \fB--ignore-arches\fR +Ignore arch-specific failures (where arch != host) +.TP +\fB-m\fR, \fB--commitmsg\fR +Adds a commit message via the command line +.TP +\fB-M\fR, \fB--commitmsgfile\fR +Adds a commit message from the specified file +.TP +\fB-V\fR, \fB--version\fR +Show version info +.TP +\fB-h\fR, \fB--help\fR +Show this screen +.SH MODES .TP .B full -Scan current directory tree for QA issues (full listing). See \fBQA KEYWORDS\fR for more -information. +Scan directory tree for QA issues (full listing) +.TP +.B last +Remember report from last run .TP .B help -Show the help screen. +Show this screen +.TP +.B scan +Scan directory tree for QA issues (default) +.TP +.B fix +Fix simple QA issues (stray digests, missing digests) +.TP +.B lfull +Remember report from last run (full listing) .TP .B commit -Scan current directory tree for QA issues. If OK, commit via cvs. +Scan directory tree for QA issues; if OK, commit via cvs .SH QA KEYWORDS .TP -.BR CVS/Entries.IO_error -While attempting to commit, a CVS/Entries directory wasn't found- either the cvs co is damaged, or permissions block repoman from reading the file. +.B CVS/Entries.IO_error +Attempting to commit, and an IO error was encountered access the Entries file .TP -.BR digest.assumed -Files listed in the digest were not found in your $DISTDIR so the existing -digest entries are assumed to be correct. +.B DEPEND.bad +User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) .TP -.BR digest.disjointed -The digests were not added to cvs when the matching ebuild had already been -added. +.B DEPEND.badindev +User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds) in developing arch .TP -.BR digest.missing -Ebuild files were found that do not have a corresponding digest. +.B DEPEND.badmasked +Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) .TP -.BR digest.notadded -The digests were found but have not been added to cvs. +.B DEPEND.badmaskedindev +Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) in developing arch .TP -.BR digest.stray -Digest files were found that do not have a corresponding ebuild. +.B DEPEND.syntax +Syntax error in DEPEND (usually an extra/missing space/parenthesis) .TP -.BR ebuild.disjointed -The ebuilds were not added to cvs when the matching digest had already been -added. +.B DESCRIPTION.missing +Ebuilds that have a missing or empty DESCRIPTION variable .TP -.BR ebuild.invalidname -The ebuild has a filename that is not-parseable by portage. +.B HOMEPAGE.missing +Ebuilds that have a missing or empty HOMEPAGE variable .TP -.BR changelog.missing -The ChangeLog file was not found. +.B IUSE.invalid +This build has a variable in IUSE that is not in the use.desc or use.local.desc file .TP -.BR ebuild.namenomatch -The euild has a filename that does not have the same name as the parent -directory. +.B KEYWORDS.invalid +This ebuild contains KEYWORDS that are not listed in profiles/arch.list or for which no valid profile was found .TP -.BR ebuild.notadded -The ebuilds were found but have not been added to cvs. +.B KEYWORDS.missing +Ebuilds that have a missing or empty KEYWORDS variable .TP -.BR file.executable -The file has the executable bit set, when it's not needed- check is limited to ebuilds, digests, Manifest, ChangeLog, and metadata.xml. +.B KEYWORDS.stupid +Ebuilds that use KEYWORDS=-* instead of package.mask .TP -.BR file.size -The file's size exceeds 20k, current policy is no files over 20k in a packages files directory. +.B LICENSE.invalid +This ebuild is listing a license that doesnt exist in portages license/ dir. .TP -.BR filedir.missing -The package lacks a files directory. +.B LICENSE.missing +Ebuilds that have a missing or empty LICENSE variable .TP -.BR DEPEND.bad -User-visible ebuilds with bad DEPEND settings (matched against *visible* ebuilds). -In other words, user-visiable ebuilds could not be found to satisfy the DEPEND settings. +.B LICENSE.syntax +Syntax error in LICENSE (usually an extra/missing space/parenthesis) .TP -.BR RDEPEND.bad -User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds). -In other words, user-visiable ebuilds could not be found to satisfy the RDEPEND settings. +.B PDEPEND.bad +User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) .TP -.BR DEPEND.badmasked -Masked ebuilds with bad DEPEND settings (matched against *all* ebuilds) -In other words, no ebuilds could not be found to satisfy the DEPEND settings. +.B PDEPEND.badindev +User-visible ebuilds with bad PDEPEND settings (matched against *visible* ebuilds) in developing arch +.TP +.B PDEPEND.badmasked +Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) +.TP +.B PDEPEND.badmaskedindev +Masked ebuilds with PDEPEND settings (matched against *all* ebuilds) in developing arch +.TP +.B PDEPEND.syntax +Syntax error in PDEPEND (usually an extra/missing space/parenthesis) .TP -.BR RDEPEND.badmasked +.B PROVIDE.syntax +Syntax error in PROVIDE (usually an extra/missing space/parenthesis) +.TP +.B RDEPEND.bad +User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) +.TP +.B RDEPEND.badindev +User-visible ebuilds with bad RDEPEND settings (matched against *visible* ebuilds) in developing arch +.TP +.B RDEPEND.badmasked Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) -In other words, no ebuilds could not be found to satisfy the RDEPEND settings. .TP -.BR ebuild.syntax -An error occured while generating the cache entry. This typically means the ebuild contains syntax error(s). +.B RDEPEND.badmaskedindev +Masked ebuilds with RDEPEND settings (matched against *all* ebuilds) in developing arch +.TP +.B RDEPEND.syntax +Syntax error in RDEPEND (usually an extra/missing space/parenthesis) +.TP +.B SLOT.missing +Ebuilds that have a missing or empty SLOT variable +.TP +.B changelog.missing +Missing ChangeLog files +.TP +.B changelog.notadded +ChangeLogs that exist but have not been added to cvs +.TP +.B digest.assumed +Existing digest must be assumed correct (Package level only) +.TP +.B digest.disjointed +Digests not added to cvs when the matching ebuild has been added +.TP +.B digest.fail +Digest does not match the specified local file +.TP +.B digest.missing +Digest files that are missing (ebuild exists, digest doesn't) +.TP +.B digest.notadded +Digests that exist but have not been added to cvs +.TP +.B digest.partial +Digest files do not contain all corresponding URI elements +.TP +.B digest.stray +Digest files that do not have a corresponding ebuild +.TP +.B digest.unmatch +Digests which are incomplete (please check if your USE/ARCH includes all files) +.TP +.B digest.unused +Digest entry has no matching SRC_URI entry +.TP +.B ebuild.allmasked +All ebuilds are masked for this package (Package level only) +.TP +.B ebuild.badheader +This ebuild has a malformed header +.TP +.B ebuild.disjointed +Ebuilds not added to cvs when the matching digest has been added +.TP +.B ebuild.invalidname +Ebuild files with a non-parseable or syntactically incorrect name (or using 2.1 versioning extensions) +.TP +.B ebuild.majorsyn +This ebuild has a major syntax error that may cause the ebuild to fail partially or fully +.TP +.B ebuild.minorsyn +This ebuild has a minor syntax error that contravenes gentoo coding style +.TP +.B ebuild.namenomatch +Ebuild files that do not have the same name as their parent directory +.TP +.B ebuild.nesteddie +Placing 'die' inside ( ) prints an error, but doesn't stop the ebuild. +.TP +.B ebuild.nostable +There are no ebuilds that are marked as stable for your ARCH +.TP +.B ebuild.notadded +Ebuilds that exist but have not been added to cvs +.TP +.B ebuild.output +A simple sourcing of the ebuild produces output; this breaks ebuild policy. +.TP +.B ebuild.syntax +Error generating cache entry for ebuild; typically caused by ebuild syntax error +.TP +.B file.UTF8 +File is not UTF8 compliant +.TP +.B file.executable +Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit +.TP +.B file.name +File/dir name must be composed of only the following chars: a-zA-Z0-9._-+: +.TP +.B file.size +Files in the files directory must be under 20k +.TP +.B filedir.missing +Package lacks a files directory .TP -.BR ebuild.output -A simple sourcing of the ebuild produces output. This breaks ebuild policy. +.B metadata.bad +Bad metadata.xml files .TP -.BR IUSE.invalid -The ebuild contains a useflag in IUSE that is not in use.desc or -use.local.desc. You should add a description of the USE flag prior to -commiting the ebuild. +.B metadata.missing +Missing metadata.xml files .TP -.BR KEYWORDS.missing -The ebuild does not have a KEYWORDS variable. +.B usage.obsolete +The ebuild makes use of an obsolete construct .TP -.BR KEYWORDS.stupid -The ebuild uses KEYWORDS=-* instead of package.mask. +.B variable.readonly +Assigning a readonly variable .TP -.BR LICENSE.missing -The ebuild does not have a LICENSE variable. +.B virtual.exists +PROVIDE contains existing package names .TP -.BR DESCRIPTION.missing -The ebuild does not have a DESCRIPTION variable. +.B virtual.unavailable +PROVIDE contains a virtual which contains no profile default .TP -.BR SLOT.missing -The ebuild does not have a SLOT variable. +.B virtual.versioned +PROVIDE contains virtuals with versions .SH "REPORTING BUGS" Please report bugs via http://bugs.gentoo.org/ .SH AUTHORS diff --git a/pym/getbinpkg.py b/pym/getbinpkg.py index ccffc1dd2..d9c9810f5 100644 --- a/pym/getbinpkg.py +++ b/pym/getbinpkg.py @@ -5,7 +5,12 @@ from output import * -import htmllib,HTMLParser,string,formatter,sys,os,xpak,time,tempfile,cPickle,base64 +import htmllib,HTMLParser,string,formatter,sys,os,xpak,time,tempfile,base64 + +try: + import cPickle +except ImportError: + import pickle as cPickle try: import ftplib diff --git a/pym/portage.py b/pym/portage.py index 64e9d85c1..290b6af16 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -19,10 +19,14 @@ except: raise SystemExit, 127 try: - import os,string,types,atexit,signal,fcntl - import time,cPickle,traceback,copy + import os,string,types,signal,fcntl,errno + import time,traceback,copy import re,pwd,grp,commands import shlex,shutil + try: + import cPickle + except ImportError: + import pickle as cPickle import stat import commands @@ -88,13 +92,15 @@ try: portage_uid, portage_gid import portage_util - from portage_util import grabdict, grabdict_package, grabfile, grabfile_package, \ + from portage_util import atomic_ofstream, dump_traceback, getconfig, grabdict, \ + grabdict_package, grabfile, grabfile_package, \ map_dictlist_vals, pickle_read, pickle_write, stack_dictlist, stack_dicts, stack_lists, \ - unique_array, varexpand, writedict, writemsg, writemsg_stdout, getconfig, dump_traceback + unique_array, varexpand, writedict, writemsg, writemsg_stdout, write_atomic import portage_exception import portage_gpg import portage_locks import portage_exec + from portage_exec import atexit_register, run_exitfuncs from portage_locks import unlockfile,unlockdir,lockfile,lockdir import portage_checksum from portage_checksum import perform_md5,perform_checksum,prelink_capable @@ -285,7 +291,7 @@ def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelis if ftype is None: ftype=[] - if not filesonly and not recursive: + if not (filesonly or dirsonly or recursive): return list if recursive: @@ -595,7 +601,7 @@ def env_update(makelinks=1): newld=specials["LDPATH"] if (oldld!=newld): #ld.so.conf needs updating and ldconfig needs to be run - myfd=open(root+portage_const.PREFIX+"/etc/ld.so.conf","w") + myfd = atomic_ofstream(os.path.join(root, portage_const.PREFIX, "etc", "ld.so.conf")) myfd.write("# ld.so.conf autogenerated by env-update; make all changes to\n") myfd.write("# contents of "+portage_const.PREFIX+"/etc/env.d directory\n") for x in specials["LDPATH"]: @@ -605,7 +611,7 @@ def env_update(makelinks=1): # Update prelink.conf if we are prelink-enabled if prelink_capable: - newprelink=open(root+portage_const.PREFIX+"/etc/prelink.conf","w") + newprelink = atomic_ofstream(os.path.join(root, portage_const.PREFIX, "etc", "prelink.conf")) newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n") newprelink.write("# contents of "+portage_const.PREFIX+"/etc/env.d directory\n") @@ -674,11 +680,11 @@ def env_update(makelinks=1): penvnotice = "# THIS FILE IS AUTOMATICALLY GENERATED BY env-update.\n" penvnotice += "# DO NOT EDIT THIS FILE. CHANGES TO STARTUP PROFILES\n" cenvnotice = penvnotice[:] - penvnotice += "# GO INTO /etc/profile NOT /etc/profile.env\n\n" - cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n" + penvnotice += "# GO INTO "+portage_const.PREFIX+"/etc/profile NOT "+portage_const.PREFIX+"/etc/profile.env\n\n" + cenvnotice += "# GO INTO "+portage_const.PREFIX"+/etc/csh.cshrc NOT "+portage_const.PREFIX+"/etc/csh.env\n\n" #create /etc/profile.env for bash support - outfile=open(root+portage_const.PREFIX+"/etc/profile.env","w") + outfile = atomic_ofstream(os.path.join(root, portage_const.PREFIX, "etc", "profile.env")) outfile.write(penvnotice) for path in specials.keys(): @@ -702,7 +708,7 @@ def env_update(makelinks=1): outfile.close() #create /etc/csh.env for (t)csh support - outfile=open(root+portage_const.PREFIX+"/etc/csh.env","w") + outfile = atomic_ofstream(os.path.join(root, portage_const.PREFIX, "etc", "csh.env")) outfile.write(cenvnotice) for path in specials.keys(): @@ -1839,43 +1845,31 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", print "!!! No write access to %s" % mysettings["DISTDIR"]+"/" can_fetch=False else: - mystat=os.stat(mysettings["DISTDIR"]+"/") - if mystat.st_gid != portage_gid: + def distdir_perms(filename): try: - os.chown(mysettings["DISTDIR"],-1,portage_gid) + portage_util.apply_permissions(filename, gid=portage_gid, mode=0775) except OSError, oe: - if oe.errno == 1: - print red("!!!")+" Unable to chgrp of %s to portage, continuing\n" % mysettings["DISTDIR"] + if oe.errno == errno.EPERM: + writemsg("!!! Unable to apply group permissions to '%s'. Non-root users may experience issues.\n" + % filename) else: raise oe - - # writable by portage_gid? This is specific to root, adjust perms if needed automatically. - if not stat.S_IMODE(mystat.st_mode) & 020: + distdir_perms(mysettings["DISTDIR"]) + if use_locks and locks_in_subdir: + distlocks_subdir = os.path.join(mysettings["DISTDIR"], locks_in_subdir) try: - os.chmod(mysettings["DISTDIR"],stat.S_IMODE(mystat.st_mode) | 020) + distdir_perms(distlocks_subdir) except OSError, oe: - if oe.errno == 1: - print red("!!!")+" Unable to chmod %s to perms 0755. Non-root users will experience issues.\n" % mysettings["DISTDIR"] + if oe.errno == errno.ENOENT: + os.mkdir(distlocks_subdir) + distdir_perms(distlocks_subdir) else: raise oe - - if use_locks and locks_in_subdir: - if os.path.exists(mysettings["DISTDIR"]+"/"+locks_in_subdir): - if not os.access(mysettings["DISTDIR"]+"/"+locks_in_subdir,os.W_OK): - writemsg("!!! No write access to write to %s. Aborting.\n" % mysettings["DISTDIR"]+"/"+locks_in_subdir) - return 0 - else: - old_umask=os.umask(0002) - os.mkdir(mysettings["DISTDIR"]+"/"+locks_in_subdir,0775) - if os.stat(mysettings["DISTDIR"]+"/"+locks_in_subdir).st_gid != portage_gid: - try: - os.chown(mysettings["DISTDIR"]+"/"+locks_in_subdir,-1,portage_gid) - except SystemExit, e: - raise - except: - pass - os.umask(old_umask) - + if not os.access(distlocks_subdir, os.W_OK): + writemsg("!!! No write access to write to %s. Aborting.\n" % distlocks_subdir) + return 0 + del distlocks_subdir + del distdir_perms for myfile in filedict.keys(): fetched=0 @@ -2688,8 +2682,8 @@ def doebuild(myebuild,mydo,myroot,mysettings,debug=0,listonly=0,fetchonly=0,clea for x in listdir(mysettings["CONFCACHE_DIR"]): p = os.path.join(mysettings["CONFCACHE_DIR"], x) st = os.stat(p) - if not (st.st_mode & 07777) & 07600 == 0600: - os.chmod(p, (st.st_mode & 0777) | 0600) + if not (st.st_mode & 07777) & 07660 == 0660: + os.chmod(p, (st.st_mode & 0777) | 0660) if not st.st_gid == portage_gid: os.chown(p, -1, portage_gid) @@ -3031,7 +3025,6 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None): except SystemExit, e: raise except Exception, e: - import errno if e[0]!=errno.EXDEV: # Some random error. print "!!! Failed to move",src,"to",dest @@ -3666,7 +3659,8 @@ def fixdbentries(old_value, new_value, dbdir): """python replacement for the fixdbentries script, replaces old_value with new_value for package names in files in dbdir.""" for myfile in [f for f in os.listdir(dbdir) if not f == "CONTENTS"]: - f = open(dbdir+"/"+myfile, "r") + file_path = os.path.join(dbdir, myfile) + f = open(file_path, "r") mycontent = f.read() f.close() if not mycontent.count(old_value): @@ -3676,9 +3670,7 @@ def fixdbentries(old_value, new_value, dbdir): mycontent = re.sub(old_value+"(\\s)", new_value+"\\1", mycontent) mycontent = re.sub(old_value+"(-[^a-zA-Z])", new_value+"\\1", mycontent) mycontent = re.sub(old_value+"([^a-zA-Z0-9-])", new_value+"\\1", mycontent) - f = open(dbdir+"/"+myfile, "w") - f.write(mycontent) - f.close() + write_atomic(file_path, mycontent) class packagetree: def __init__(self,virtual,clone=None): @@ -4141,12 +4133,7 @@ class dbapi: #increment counter counter += 1 # update new global counter file - newcpath=cpath+".new" - newcfile=open(newcpath,"w") - newcfile.write(str(counter)) - newcfile.close() - # now move global counter file into place - os.rename(newcpath,cpath) + write_atomic(cpath, str(counter)) return counter def invalidentry(self, mypath): @@ -4291,10 +4278,7 @@ class vardbapi(dbapi): if len(myl) == 1: try: # Only one package... Counter doesn't matter. - myf = open(cpath, "w") - myf.write("1") - myf.flush() - myf.close() + write_atomic(cpath, "1") counter = 1 except SystemExit, e: raise @@ -4314,13 +4298,8 @@ class vardbapi(dbapi): else: counter=long(0) if corrupted: - newcpath=cpath+".new" # update new global counter file - newcfile=open(newcpath,"w") - newcfile.write(str(counter)) - newcfile.close() - # now move global counter file into place - os.rename(newcpath,cpath) + write_atomic(cpath, str(counter)) return counter def cpv_inject(self,mycpv): @@ -4328,9 +4307,7 @@ class vardbapi(dbapi): os.makedirs(self.root+VDB_PATH+"/"+mycpv) counter=db[self.root]["vartree"].dbapi.counter_tick(self.root,mycpv=mycpv) # write local package counter so that emerge clean does the right thing - lcfile=open(self.root+VDB_PATH+"/"+mycpv+"/COUNTER","w") - lcfile.write(str(counter)) - lcfile.close() + write_atomic(os.path.join(self.root, VDB_PATH, mycpv, "COUNTER"), str(counter)) def isInjected(self,mycpv): if self.cpv_exists(mycpv): @@ -4343,6 +4320,10 @@ class vardbapi(dbapi): def move_ent(self,mylist): origcp=mylist[1] newcp=mylist[2] + # sanity check + for cp in [origcp,newcp]: + if not (isvalidatom(cp) and isjustname(cp)): + raise portage_exception.InvalidPackageName(cp) origmatches=self.match(origcp,use_cache=0) if not origmatches: return @@ -4375,9 +4356,7 @@ class vardbapi(dbapi): if os.path.exists(old_eb_path+".ebuild"): os.rename(old_eb_path+".ebuild", new_eb_path+".ebuild") - catfile=open(newpath+"/CATEGORY", "w") - catfile.write(mynewcat+"\n") - catfile.close() + write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") dbdir = self.root+VDB_PATH for catdir in listdir(dbdir): @@ -4393,7 +4372,11 @@ class vardbapi(dbapi): origslot=mylist[2] newslot=mylist[3] + if not isvalidatom(pkg): + raise portage_exception.InvalidAtom(pkg) + origmatches=self.match(pkg,use_cache=0) + if not origmatches: return for mycpv in origmatches: @@ -4409,9 +4392,7 @@ class vardbapi(dbapi): continue writemsg("s") - slotfile=open(origpath+"/SLOT", "w") - slotfile.write(newslot+"\n") - slotfile.close() + write_atomic(os.path.join(origpath, "SLOT"), newslot+"\n") def cp_list(self,mycp,use_cache=1): mysplit=mycp.split("/") @@ -5282,6 +5263,10 @@ class binarytree(packagetree): self.populate() origcp=mylist[1] newcp=mylist[2] + # sanity check + for cp in [origcp,newcp]: + if not (isvalidatom(cp) and isjustname(cp)): + raise portage_exception.InvalidPackageName(cp) mynewcat=newcp.split("/")[0] origmatches=self.dbapi.cp_list(origcp) if not origmatches: @@ -5314,9 +5299,7 @@ class binarytree(packagetree): fixdbentries(origcp, newcp, mytmpdir) - catfile=open(mytmpdir+"/CATEGORY", "w") - catfile.write(mynewcat+"\n") - catfile.close() + write_atomic(os.path.join(mytmpdir, "CATEGORY"), mynewcat+"\n") try: os.rename(mytmpdir+"/"+string.split(mycpv,"/")[1]+".ebuild", mytmpdir+"/"+string.split(mynewcpv, "/")[1]+".ebuild") except SystemExit, e: @@ -5340,6 +5323,10 @@ class binarytree(packagetree): pkg=mylist[1] origslot=mylist[2] newslot=mylist[3] + + if not isvalidatom(pkg): + raise portage_exception.InvalidAtom(pkg) + origmatches=self.dbapi.match(pkg) if not origmatches: return @@ -5365,9 +5352,7 @@ class binarytree(packagetree): sys.stdout.write("S") sys.stdout.flush() - slotfile=open(mytmpdir+"/SLOT", "w") - slotfile.write(newslot+"\n") - slotfile.close() + write_atomic(os.path.join(mytmpdir, "SLOT"), newslot+"\n") mytbz2.recompose(mytmpdir, cleanup=1) return 1 @@ -5741,16 +5726,27 @@ class dblink: #process symlinks second-to-last, directories last. mydirs=[] modprotect="/lib/modules/" - for obj in mykeys: - obj=os.path.normpath(obj) + for objkey in mykeys: + obj=os.path.normpath(objkey) if obj[:2]=="//": obj=obj[1:] - if not os.path.exists(obj): - if not os.path.islink(obj): + statobj = None + try: + statobj = os.stat(obj) + except OSError: + pass + lstatobj = None + try: + lstatobj = os.lstat(obj) + except (OSError, AttributeError): + pass + islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode) + if statobj is None: + if not islink: #we skip this if we're dealing with a symlink - #because os.path.exists() will operate on the + #because os.stat() will operate on the #link target rather than the link itself. - writemsg_stdout("--- !found "+str(pkgfiles[obj][0])+ " %s\n" % obj) + writemsg_stdout("--- !found "+str(pkgfiles[objkey][0])+ " %s\n" % obj) continue # next line includes a tweak to protect modules from being unmerged, # but we don't protect modules from being overwritten if they are @@ -5758,56 +5754,61 @@ class dblink: # functionality for /lib/modules. For portage-ng both capabilities # should be able to be independently specified. if self.isprotected(obj) or ((len(obj) > len(modprotect)) and (obj[0:len(modprotect)]==modprotect)): - writemsg_stdout("--- cfgpro %s %s\n" % (pkgfiles[obj][0], obj)) + writemsg_stdout("--- cfgpro %s %s\n" % (pkgfiles[objkey][0], obj)) continue - lstatobj=os.lstat(obj) lmtime=str(lstatobj[stat.ST_MTIME]) - if (pkgfiles[obj][0] not in ("dir","fif","dev")) and (lmtime != pkgfiles[obj][1]): - writemsg_stdout("--- !mtime %s %s\n" % (pkgfiles[obj][0], obj)) + if (pkgfiles[objkey][0] not in ("dir","fif","dev")) and (lmtime != pkgfiles[objkey][1]): + writemsg_stdout("--- !mtime %s %s\n" % (pkgfiles[objkey][0], obj)) continue - if pkgfiles[obj][0]=="dir": - if not os.path.isdir(obj): - writemsg_stdout("--- !dir %s %s\n" % ("dir", obj)) + if pkgfiles[objkey][0]=="dir": + if statobj is None or not stat.S_ISDIR(statobj.st_mode): + writemsg_stdout("--- !dir %s %s\n" % ("dir", obj)) continue mydirs.append(obj) - elif pkgfiles[obj][0]=="sym": - if not os.path.islink(obj): - writemsg_stdout("--- !sym %s %s\n" % ("sym", obj)) + elif pkgfiles[objkey][0]=="sym": + if not islink: + writemsg_stdout("--- !sym %s %s\n" % ("sym", obj)) continue try: os.unlink(obj) - writemsg_stdout("<<< %s %s\n" % ("sym",obj)) + writemsg_stdout("<<< %s %s\n" % ("sym",obj)) except (OSError,IOError),e: - writemsg_stdout("!!! %s %s\n" % ("sym",obj)) - elif pkgfiles[obj][0]=="obj": - if not os.path.isfile(obj): - writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) + writemsg_stdout("!!! %s %s\n" % ("sym",obj)) + elif pkgfiles[objkey][0]=="obj": + if statobj is None or not stat.S_ISREG(statobj.st_mode): + writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) + continue + mymd5 = None + try: + mymd5 = portage_checksum.perform_md5(obj, calc_prelink=1) + except portage_exception.FileNotFound, e: + # the file has disappeared between now and our stat call + writemsg_stdout("--- !obj %s %s\n" % ("obj", obj)) continue - mymd5=portage_checksum.perform_md5(obj, calc_prelink=1) # string.lower is needed because db entries used to be in upper-case. The # string.lower allows for backwards compatibility. - if mymd5 != string.lower(pkgfiles[obj][2]): - writemsg_stdout("--- !md5 %s %s\n" % ("obj", obj)) + if mymd5 != string.lower(pkgfiles[objkey][2]): + writemsg_stdout("--- !md5 %s %s\n" % ("obj", obj)) continue try: os.unlink(obj) except (OSError,IOError),e: pass - writemsg_stdout("<<< %s %s\n" % ("obj",obj)) - elif pkgfiles[obj][0]=="fif": + writemsg_stdout("<<< %s %s\n" % ("obj",obj)) + elif pkgfiles[objkey][0]=="fif": if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): - writemsg_stdout("--- !fif %s %s\n" % ("fif", obj)) + writemsg_stdout("--- !fif %s %s\n" % ("fif", obj)) continue try: os.unlink(obj) except (OSError,IOError),e: pass - writemsg_stdout("<<< %s %s\n" % ("fif",obj)) - elif pkgfiles[obj][0]=="dev": - writemsg_stdout("--- %s %s\n" % ("dev",obj)) + writemsg_stdout("<<< %s %s\n" % ("fif",obj)) + elif pkgfiles[objkey][0]=="dev": + writemsg_stdout("--- %s %s\n" % ("dev",obj)) mydirs.sort() mydirs.reverse() @@ -5817,7 +5818,7 @@ class dblink: if not last_non_empty.startswith(obj) and not listdir(obj): try: os.rmdir(obj) - writemsg_stdout("<<< %s %s\n" % ("dir",obj)) + writemsg_stdout("<<< %s %s\n" % ("dir",obj)) last_non_empty = "" continue except (OSError,IOError),e: @@ -5862,10 +5863,7 @@ class dblink: os.chown(pdir, 0, portage_gid) os.chmod(pdir, 02770) - myworld=open(self.myroot+WORLD_FILE,"w") - for x in newworldlist: - myworld.write(x+"\n") - myworld.close() + write_atomic(os.path.join(self.myroot,WORLD_FILE),"\n".join(newworldlist)) #do original postrm if myebuildpath and os.path.exists(myebuildpath): @@ -6849,8 +6847,12 @@ def do_upgrade(mykey): sys.stdout.flush() if mysplit[0]=="move": - db["/"]["vartree"].dbapi.move_ent(mysplit) - db["/"]["bintree"].move_ent(mysplit) + try: + db["/"]["vartree"].dbapi.move_ent(mysplit) + db["/"]["bintree"].move_ent(mysplit) + except portage_exception.InvalidPackageName, e: + writemsg("\nERROR: Malformed update entry '%s'\n" % myline) + continue #update world entries: for x in range(0,len(worldlist)): #update world entries, if any. @@ -6870,8 +6872,11 @@ def do_upgrade(mykey): sys.stdout.flush() elif mysplit[0]=="slotmove": - db["/"]["vartree"].dbapi.move_slot_ent(mysplit) - db["/"]["bintree"].move_slot_ent(mysplit,settings["PORTAGE_TMPDIR"]+"/tbz2") + try: + db["/"]["vartree"].dbapi.move_slot_ent(mysplit) + db["/"]["bintree"].move_slot_ent(mysplit,settings["PORTAGE_TMPDIR"]+"/tbz2") + except portage_exception.InvalidAtom, e: + writemsg("\nERROR: Malformed update entry '%s'\n" % myline) for x in update_files: mydblink = dblink('','','/',settings) @@ -6880,9 +6885,7 @@ def do_upgrade(mykey): else: updating_file=USER_CONFIG_PATH+os.path.sep+x try: - myfile=open(updating_file,"w") - myfile.writelines(file_contents[x]) - myfile.close() + write_atomic(updating_file, "".join(file_contents[x])) except IOError: continue @@ -6896,23 +6899,24 @@ def do_upgrade(mykey): if processed: #update our internal mtime since we processed all our directives. mtimedb["updates"][mykey]=os.stat(mykey)[stat.ST_MTIME] - myworld=open("/"+WORLD_FILE,"w") - for x in worldlist: - myworld.write(x+"\n") - myworld.close() + write_atomic(WORLD_FILE,"\n".join(worldlist)) print "" def commit_mtimedb(): if mtimedb: # Store mtimedb mymfn=mtimedbfile + f = None try: mtimedb["version"]=VERSION - cPickle.dump(mtimedb, open(mymfn,"w"), -1) + f = atomic_ofstream(mymfn) + cPickle.dump(mtimedb, f, -1) + f.close() except SystemExit, e: raise except Exception, e: - pass + if f is not None: + f.abort() try: os.chown(mymfn,uid,portage_gid) @@ -6928,7 +6932,7 @@ def portageexit(): close_portdbapi_caches() commit_mtimedb() -atexit.register(portageexit) +atexit_register(portageexit) if (secpass==2) and (not os.environ.has_key("SANDBOX_ACTIVE")): if settings["PORTAGE_CALLER"] in ["emerge","fixpackages"]: diff --git a/pym/portage_checksum.py b/pym/portage_checksum.py index f837966bc..2dd447df2 100644 --- a/pym/portage_checksum.py +++ b/pym/portage_checksum.py @@ -6,6 +6,7 @@ from portage_const import PRIVATE_PATH,PRELINK_BINARY,HASHING_BLOCKSIZE import os +import errno import shutil import stat import portage_exception @@ -130,12 +131,21 @@ def perform_checksum(filename, hash_function=md5hash, calc_prelink=0): if retval==0: #portage_util.writemsg(">>> prelink checksum '"+str(filename)+"'.\n") myfilename=prelink_tmpfile - - myhash, mysize = hash_function(myfilename) - + try: + myhash, mysize = hash_function(myfilename) + except (OSError, IOError), e: + if e.errno == errno.ENOENT: + raise portage_exception.FileNotFound(e) + else: + raise e if calc_prelink and prelink_capable: - if os.path.exists(prelink_tmpfile): + try: os.unlink(prelink_tmpfile) + except OSError, oe: + if oe.errno == errno.ENOENT: + pass + else: + raise oe portage_locks.unlockfile(mylock) return (myhash,mysize) diff --git a/pym/portage_data.py b/pym/portage_data.py index b840541ff..cecdbfc59 100644 --- a/pym/portage_data.py +++ b/pym/portage_data.py @@ -20,7 +20,7 @@ elif ostype == "Darwin": os.environ["XARGS"]="xargs" def lchown(*pos_args, **key_args): pass -elif ostype in ["FreeBSD","OpenBSD","NetBSD"]: +elif ostype.endswith("BSD") or ostype =="DragonFly": userland="BSD" os.environ["XARGS"]="xargs" elif ostype in ["BeOS","Haiku","HP-UX","SunOS"]: diff --git a/pym/portage_exception.py b/pym/portage_exception.py index d810876f1..ae3a9db4f 100644 --- a/pym/portage_exception.py +++ b/pym/portage_exception.py @@ -58,6 +58,9 @@ class PackageNotFound(PortagePackageException): class InvalidPackageName(PortagePackageException): """Malformed package name""" +class InvalidAtom(PortagePackageException): + """Malformed atom spec""" + class UnsupportedAPIException(PortagePackageException): """Unsupported API""" def __init__(self, cpv, api): diff --git a/pym/portage_exec.py b/pym/portage_exec.py index 72621ef1d..08614ed32 100644 --- a/pym/portage_exec.py +++ b/pym/portage_exec.py @@ -7,6 +7,7 @@ import os, atexit, signal, sys import portage_data +from portage_util import dump_traceback from portage_const import BASH_BINARY, SANDBOX_BINARY @@ -39,6 +40,39 @@ def spawn_sandbox(mycommand, opt_name=None, **keywords): args.append(mycommand) return spawn(args, opt_name=opt_name, **keywords) +_exithandlers = [] +def atexit_register(func, *args, **kargs): + """Wrapper around atexit.register that is needed in order to track + what is registered. For example, when portage restarts itself via + os.execv, the atexit module does not work so we have to do it + manually by calling the run_exitfuncs() function in this module.""" + _exithandlers.append((func, args, kargs)) + +def run_exitfuncs(): + """This should behave identically to the routine performed by + the atexit module at exit time. It's only necessary to call this + function when atexit will not work (because of os.execv, for + example).""" + + # This function is a copy of the private atexit._run_exitfuncs() + # from the python 2.4.2 sources. The only difference from the + # original function is in the output to stderr. + exc_info = None + while _exithandlers: + func, targs, kargs = _exithandlers.pop() + try: + func(*targs, **kargs) + except SystemExit: + exc_info = sys.exc_info() + except: + dump_traceback("Error in portage_exec.run_exitfuncs", noiselevel=0) + exc_info = sys.exc_info() + + if exc_info is not None: + raise exc_info[0], exc_info[1], exc_info[2] + +atexit.register(run_exitfuncs) + # We need to make sure that any processes spawned are killed off when # we exit. spawn() takes care of adding and removing pids to this list # as it creates and cleans up processes. @@ -55,7 +89,7 @@ def cleanup(): # of spawn(). pass -atexit.register(cleanup) +atexit_register(cleanup) def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, logfile=None, diff --git a/pym/portage_locks.py b/pym/portage_locks.py index 1bb6c4c88..ee2e0b0d4 100644 --- a/pym/portage_locks.py +++ b/pym/portage_locks.py @@ -4,7 +4,6 @@ # $Id: /var/cvsroot/gentoo-src/portage/pym/portage_locks.py,v 1.18.2.2 2005/01/16 02:35:33 carpaski Exp $ -import atexit import errno import os import stat @@ -15,6 +14,7 @@ import portage_exception import portage_file import portage_util import portage_data +from portage_exec import atexit_register from portage_localization import _ import portage_const @@ -31,7 +31,7 @@ def add_hardlock_file_to_cleanup(path): if os.path.isdir(mypath): hardlock_path_list = mypath[:] -atexit.register(clean_my_hardlocks) +atexit_register(clean_my_hardlocks) def lockdir(mydir): return lockfile(mydir,wantnewlockfile=1) @@ -123,7 +123,7 @@ def lockfile(mypath,wantnewlockfile=0,unlinkfile=0): raise - if type(lockfilename) == types.StringType and not os.path.exists(lockfilename): + if type(lockfilename) == types.StringType and os.fstat(myfd).st_nlink != 1: # The file was deleted on us... Keep trying to make one... os.close(myfd) portage_util.writemsg("lockfile recurse\n",1) @@ -148,9 +148,10 @@ def unlockfile(mytuple): unhardlink_lockfile(lockfilename) return True + # myfd may be None here due to myfd = mypath in lockfile() if type(lockfilename) == types.StringType and not os.path.exists(lockfilename): portage_util.writemsg("lockfile does not exist '%s'\n" % lockfilename,1) - if (myfd != None) and type(lockfilename) == types.StringType: + if myfd is not None: os.close(myfd) return False @@ -179,9 +180,14 @@ def unlockfile(mytuple): # We can safely delete the file. portage_util.writemsg("Got the lockfile...\n",1) #portage_util.writemsg("Unlinking...\n") - os.unlink(lockfilename) - portage_util.writemsg("Unlinked lockfile...\n",1) - locking_method(myfd,fcntl.LOCK_UN) + if os.fstat(myfd).st_nlink == 1: + os.unlink(lockfilename) + portage_util.writemsg("Unlinked lockfile...\n",1) + locking_method(myfd,fcntl.LOCK_UN) + else: + portage_util.writemsg("lockfile does not exist '%s'\n" % lockfilename,1) + os.close(myfd) + return False except SystemExit, e: raise except Exception, e: diff --git a/pym/portage_util.py b/pym/portage_util.py index ef155b5db..c8d55fcdd 100644 --- a/pym/portage_util.py +++ b/pym/portage_util.py @@ -4,6 +4,11 @@ import sys,string,shlex,os.path +try: + import cPickle +except ImportError: + import pickle as cPickle + if not hasattr(__builtins__, "set"): from sets import Set as set @@ -189,9 +194,8 @@ def writedict(mydict,myfilename,writekey=True): """Writes out a dict to a file; writekey=0 mode doesn't write out the key and assumes all values are strings, not lists.""" myfile = None - myf2 = "%s.%i" % (myfilename, os.getpid()) try: - myfile=open(myf2,"w") + myfile = atomic_ofstream(myfilename) if not writekey: for x in mydict.values(): myfile.write(x+"\n") @@ -199,11 +203,9 @@ def writedict(mydict,myfilename,writekey=True): for x in mydict.keys(): myfile.write("%s %s\n" % (x, " ".join(mydict[x]))) myfile.close() - os.rename(myf2, myfilename) - except IOError: if myfile is not None: - os.unlink(myf2) + myfile.abort() return 0 return 1 @@ -365,7 +367,7 @@ def varexpand(mystring,mydict={}): return newstring[1:] def pickle_write(data,filename,debug=0): - import cPickle,os + import os try: myf=open(filename,"w") cPickle.dump(data,myf,-1) @@ -381,7 +383,7 @@ def pickle_write(data,filename,debug=0): return 1 def pickle_read(filename,default=None,debug=0): - import cPickle,os + import os if not os.access(filename, os.R_OK): writemsg("pickle_read(): File not readable. '"+filename+"'\n",1) return default @@ -451,4 +453,89 @@ def unique_array(s): if x not in u: u.append(x) return u - + +def apply_permissions(filename, uid=-1, gid=-1, mode=0, + stat_cached=None): + """Apply user, group, and mode bits to a file + if the existing bits do not already match.""" + + if stat_cached is None: + stat_cached = os.stat(filename) + + if (uid != -1 and uid != stat_cached.st_uid) or \ + (gid != -1 and gid != stat_cached.st_gid): + os.chown(filename, uid, gid) + + if mode & stat_cached.st_mode != mode: + os.chmod(filename, mode | stat_cached.st_mode) + +def apply_stat_permissions(filename, newstat, stat_cached=None): + """wrapper around apply_permissions that gets + uid, gid, and mode from a stat object""" + apply_permissions(filename, uid=newstat.st_uid, gid=newstat.st_gid, + mode=newstat.st_mode, stat_cached=stat_cached) + +class atomic_ofstream(file): + """Write a file atomically via os.rename(). Atomic replacement prevents + interprocess interference and prevents corruption of the target + file when the write is interrupted (for example, when an 'out of space' + error occurs).""" + + def __init__(self, filename, mode='w', **kargs): + """Opens a temporary filename.pid in the same directory as filename.""" + self._aborted = False + self._real_name = filename + tmp_name = "%s.%i" % (filename, os.getpid()) + super(atomic_ofstream, self).__init__(tmp_name, mode=mode, **kargs) + + def close(self): + """Closes the temporary file, copies permissions (if possible), + and performs the atomic replacement via os.rename(). If the abort() + method has been called, then the temp file is closed and removed.""" + if not self.closed: + try: + super(atomic_ofstream, self).close() + if not self._aborted: + try: + apply_stat_permissions(self.name, os.stat(self._real_name)) + except OSError, oe: + import errno + if oe.errno in (errno.ENOENT,errno.EPERM): + pass + else: + raise oe + os.rename(self.name, self._real_name) + finally: + # Make sure we cleanup the temp file + # even if an exception is raised. + try: + os.unlink(self.name) + except OSError, oe: + pass + + def abort(self): + """If an error occurs while writing the file, the user should + call this method in order to leave the target file unchanged. + This will call close() automatically.""" + if not self._aborted: + self._aborted = True + self.close() + + def __del__(self): + """If the user does not explicitely call close(), it is + assumed that an error has occurred, so we abort().""" + if not self.closed: + self.abort() + # ensure destructor from the base class is called + base_destructor = getattr(super(atomic_ofstream, self), '__del__', None) + if base_destructor is not None: + base_destructor() + +def write_atomic(file_path, content): + f = atomic_ofstream(file_path) + try: + f.write(content) + f.close() + except IOError, ioe: + f.abort() + raise ioe -- 2.26.2