From: Fabian Groffen Date: Fri, 19 Jan 2007 19:31:53 +0000 (-0000) Subject: Merged from trunk 5673:5697 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=ccfdff7c49f1b7a82c766705e576e894956a4efa;p=portage.git Merged from trunk 5673:5697 For consistency, respect the use_binaries flag inside _expand_new_virtuals(). Allow --usepkgonly/--getbinpkgonly to work with installed deps when binpkgs aren't available for some packages. Thanks to Brent Baude for reporting. Use the portable -o option for tar. Thanks to Timothy Redaelli for the patch. make it executable (SpankMan) make sure we rename links and not just what they point to #162451 (SpankMan) Prevent packages that are installed but masked from being incorrectly flagged as "available" (unless --usepkgonly is enabled, which currently has no package.mask support). For --usepkgonly, allow _expand_new_virtuals() to fall back to the vardb in case all binaries aren't available. Fix elimination of duplicate virtuals between the binpkg repo and installed packages. Remove one last reference to ${tarvars}. For bug #162404, spawn tee outside the sesandbox domain so that it reads from a pipe between two domains. Use elif where appropriate. Thanks to ferringb. start documenting qa notices so antarus stops whining (SpankMan) superh supports sane exec stack stuff now (SpankMan) arm supports sane exec stack stuff now (SpankMan) add try finally blocks around locking for news (WarnerBro) svn path=/main/branches/prefix/; revision=5727 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 10b561fcf..1a573d859 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -536,13 +536,6 @@ unpack() { local x local y local myfail - local tarvars - - if [ "$USERLAND" == "BSD" ]; then - tarvars="" - else - tarvars="--no-same-owner" - fi [ -z "$*" ] && die "Nothing passed to the 'unpack' command" @@ -564,13 +557,13 @@ unpack() { myfail="failure unpacking ${x}" case "${x##*.}" in tar) - tar xf "${srcdir}${x}" ${tarvars} || die "$myfail" + tar xof "${srcdir}${x}" || die "$myfail" ;; tgz) - tar xzf "${srcdir}${x}" ${tarvars} || die "$myfail" + tar xozf "${srcdir}${x}" || die "$myfail" ;; tbz|tbz2) - bzip2 -dc "${srcdir}${x}" | tar xf - ${tarvars} + bzip2 -dc "${srcdir}${x}" | tar xof - assert "$myfail" ;; ZIP|zip|jar) @@ -578,14 +571,14 @@ unpack() { ;; gz|Z|z) if [ "${y}" == "tar" ]; then - tar zxf "${srcdir}${x}" ${tarvars} || die "$myfail" + tar zoxf "${srcdir}${x}" || die "$myfail" else gzip -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" fi ;; bz2|bz) if [ "${y}" == "tar" ]; then - bzip2 -dc "${srcdir}${x}" | tar xf - ${tarvars} + bzip2 -dc "${srcdir}${x}" | tar xof - assert "$myfail" else bzip2 -dc "${srcdir}${x}" > ${x%.*} || die "$myfail" diff --git a/bin/ecompressdir b/bin/ecompressdir index 3f6fb251e..5d8c53581 100755 --- a/bin/ecompressdir +++ b/bin/ecompressdir @@ -34,7 +34,8 @@ for dir in "$@" ; do while read brokenlink ; do olddest=$(readlink "${brokenlink}") newdest="${olddest}${suffix}" - ln -snf "${newdest}" "${brokenlink}" + rm -f "${brokenlink}" + ln -snf "${newdest}" "${brokenlink}${suffix}" ((ret+=$?)) done done diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh old mode 100644 new mode 100755 diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index 231e5c514..7566f366f 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -130,7 +130,7 @@ install_qa_check() { # are supported at the moment. Keep this list in sync with # http://hardened.gentoo.org/gnu-stack.xml (Arch Status) case ${CTARGET:-${CHOST}} in - i?86*|ia64*|m68k*|s390*|x86_64*) + arm*|i?86*|ia64*|m68k*|s390*|sh*|x86_64*) # Allow devs to mark things as ignorable ... e.g. things # that are binary-only and upstream isn't cooperating ... # we allow ebuild authors to set QA_EXECSTACK_arch and diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..c829639ca --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,11 @@ +all: xhtml xhtml-nochunks + +XMLTO_FLAGS_man = -x custom.xsl +man pdf txt xhtml xhtml-nochunks: + xmlto $@ $(XMLTO_FLAGS_$@) portage.docbook + +clean distclean: + rm -f *.1 *.html + +.PHONY: all clean distclean \ + man pdf txt xhtml xhtml-nochunks diff --git a/doc/custom.xsl b/doc/custom.xsl new file mode 100644 index 000000000..e69de29bb diff --git a/doc/fragment/date b/doc/fragment/date new file mode 100644 index 000000000..e69de29bb diff --git a/doc/fragment/version b/doc/fragment/version new file mode 100644 index 000000000..f2b80a3bd --- /dev/null +++ b/doc/fragment/version @@ -0,0 +1 @@ +svn-trunk diff --git a/doc/portage.docbook b/doc/portage.docbook new file mode 100644 index 000000000..1ca868722 --- /dev/null +++ b/doc/portage.docbook @@ -0,0 +1,32 @@ + + + + + + + +]> + + + + + Portage Documentation + + + + Mike + Frysinger + vapier@gentoo.org + + + + &version; + &date; + + +&qa; + + diff --git a/doc/qa.docbook b/doc/qa.docbook new file mode 100644 index 000000000..d4c920867 --- /dev/null +++ b/doc/qa.docbook @@ -0,0 +1,346 @@ + + QA Notices + + Here we'll go over each QA notice and what you (as a developer) can do to fix + the issue. If you're a user, you should of course go + file a bug. We'll only cover the + non-obvious notices here. + + + In pretty much all cases, you should try and get these issues resolved + upstream rather than simply fixing them in our ebuilds. + + + + Scanelf: Insecure RUNPATHs + + + QA Notice: The following files contain insecure RUNPATH's + + + + Some of the ELFs that would be installed on the system have insecure dynamic + RUNPATH tags. RUNPATH tags are a hardcoded list of filesystem paths that + will be searched at runtime when the ELF is executed. If the ELF has a + world accessible directory hardcoded in it, then a malicious person can + inject code at runtime by adding their own libraries to the directory. + + + Here are some of the common problems and their solutions. + + + Libtool - old versions of libtool would use too many -rpath flags + Solution: Regenerate the autotool code + + + Perl - some versions of perl would use incorrect -rpath flags + Solution: upgrade system perl build modules + + + Crappy build system - the custom build system uses -rpath incorrectly + Solution: review the LDFLAGS in the build system and make them not suck + + + Crappy ebuild - the ebuild installs ELFs instead of using the package's build system + Solution: fix the crappy ebuild to use the package's build system + + + + + + + Scanelf: Runtime Text Relocations (TEXTRELS) + + + QA Notice: The following files contain runtime text relocations + + + + Please see the Gentoo Hardened PIC Fix Guide. + + + + + Scanelf: Executable Stack (EXECSTACK) + + + QA Notice: The following files contain executable stacks + + + + Please see the Gentoo Hardened GNU Stack Guide. + + + + + Scanelf: Missing Shared Object Name (SONAME) + + + QA Notice: The following shared libraries lack a SONAME + + + + A shared library that you would link against lacks an ELF SONAME tag. With + simpler libraries, this can be acceptable, but with any sort of ABI sane + setup, you need the SONAME tag. This tag is how the system linker tells the + loader what libraries a program needs at runtime. With a missing SONAME, + the linker needs to guess and with many cases, this guess will not work for + long. + + + To fix this issue, make sure the shared library is linked with the proper + flag. You will need to replace the + ... part with the actual ABI name. For example, + if the library is named libfoo.so.1.2.3, you will + probably want to specify . + + + Note that this warning only applies to shared libraries that you would link + against. It certainly does not apply to plugins that you would dynamically + load. However, plugins should not exist in the main library directory, but + rather an application specific subdirectory in the library directory. In + other words, it should be /usr/lib/app/plugin.so rather + than /usr/lib/plugin.so. + + + + + Scanelf: Missing Needed Entries + + + QA Notice: The following shared libraries lack NEEDED entries + + + + This warning comes up when a library does not actually seem to need any + other libraries in order to run. Rarely is this true as almost every + library will need at least the system C library. + + + Once you've determined that the library is indeed being generated + incorrectly, you will need to dig into the build system to make sure that + it pulls in the libraries it needs. Often times, this is because the + build system invokes the system linker (ld) directly + instead of the system compiler driver (gcc). + + + + + Absolute Symlink In Library Directory + + + QA Notice: Found an absolute symlink in a library directory + + + + If you want to use symlinks in library directories, please use either a + relative symlink or a linker script. This can cause problems when working + with cross-compiler systems or when accessing systems in a different ROOT + directory. + + + If you have a library installed into /lib/ and you want + to have it accessible in /usr/lib/, then you should + generate a linker script so that the system toolchain can handle it properly. + Please see the linker script section + for more information. + + + + + Missing Linker Script + + + QA Notice: Missing gen_usr_ldscript + + + + If you have a shared library in /lib/ and a static + library in /usr/lib/, but no linker script in + /usr/lib/, then the toolchain will choose the incorrect + version when linking. The system linker will find the static library first + and not bother searching for a dynamic version. To overcome this, you need + to use the gen_usr_ldscript function found in the + toolchain-funcs.eclass. Refer to the + man page for information on how to use it. See this + bug report for some history + on this issue. + + + + + Excessive Files in / + + + QA Notice: Excessive files found in the / partition + + + + You should not store files that are not critical to boot and recovery in + the root filesystem. This means that static libraries and libtool scripts do + not belong in the /lib/ directory. Fix your ebuild so + it does not install there. + + + + + Portage Tempdir In Libtool Scripts + + + QA Notice: ... appears to contain PORTAGE_TMPDIR paths + + + + Older versions of libtool would incorrectly record the build and/or install + directory in the libtool script (*.la). This would lead to problems when + building other things against your package as libtool would be confused by + the old paths. + + + You may be able to cheat and use the elibtoolize function + in the libtool.eclass. However, if + that does not help, you will probably need to regenerate all of the autotool + files. + + + + + Build Warning: Strict Aliasing + + + QA Notice: Package has poor programming practices which may compile + fine but exhibit random runtime failures. + ...: warning: dereferencing type-punned pointer will break strict-aliasing rules + + + + + + + + Build Warning: Implicit Declarations + + + QA Notice: Package has poor programming practices which may compile + fine but exhibit random runtime failures. + ...: warning: implicit declaration of function ... + + + + Your code is calling functions which lack prototypes. In C++, this would + have been a build failure, but C is lazy so you just get a warning. This + can be a problem as gcc has to guess at what sort of arguments a function + takes based upon how it was called and often times, this is not the same + as what the function actually takes. The function return type is also + unknown so it's just assumed to be an integer (which is often times wrong). + This can get to be a problem when the size of the types guessed do not + actually match the size of the types the function expects. Generally, this + corresponds directly to proper coding practices (and the lack thereof). + Also, by including proper prototypes, the compiler often helps by checking + types used, proper number of arguments passed, etc... + + + To fix this, just include the proper header files for the functions in + question. If the function is a package-specific one, then you may have to + create a header/function prototype for it. + + + + + Build Warning: Used Uninitialized + + + QA Notice: Package has poor programming practices which may compile + fine but exhibit random runtime failures. + ...: warning: incompatible implicit declaration of built-in function ... + + + + This means code uses a variable without actually setting it first. In other + words, the code is basically using random garbage. + + + The fix here is simple: make sure variables are initialized properly before + using them. + + + + + Build Warning: Invalid X<=Y<=Z Comparisons + + + QA Notice: Package has poor programming practices which may compile + fine but exhibit random runtime failures. + ...: warning: is used uninitialized in this function + + + + This warning crops up either when the programmer expected the expression + to work or they just forgot to use sufficient parentheses. For example, + the following code snippets are wrong (we won't get into the technical + argument of this being valid C code; just change the code to not be + ambiguous). + + if (x <= y <= z) + ...; + if (a < b <= c) + ...; + + + + To fix this, read the code to figure out what exactly the programmer meant. + + + + + Build Warning: Non-Null Required + + + QA Notice: Package has poor programming practices which may compile + fine but exhibit random runtime failures. + ...: warning: comparisons like X<=Y<=Z do not have their mathematical meaning + + + + Many functions take pointers as arguments and require that the pointer never + be NULL. To this end, you can declare function prototypes that instruct the + compiler to do simple checks to make sure people do not incorrectly call the + function with NULL values. This warning pops up when someone calls a + function and they use NULL when they should not. Depending on the library, + the function may actually crash (they told you not to use NULL after-all, so + it's your fault :P). + + + You will need to read the code and fix it so that it does not incorrectly + call the relevant functions with NULL values. + + + + + Build Warning: Truncating Pointers + + + QA Notice: Package has poor programming practices which may compile + but will almost certainly crash on 64bit architectures. + + + + A large portion of code in the open source world is developed on the 32bit + x86 architecture. Unfortunately, this has led to many pieces of code not + handling pointer types properly. When compiled and run on a 64bit + architecture, the code in question will probably crash horribly. Some + common examples are assuming that an integer type is large enough to hold + pointers. This is true on 32bit architectures (an integer can hold 32bits + and a pointer is 32bits big), but not true on 64bit architectures (an + integer still holds just 32bits, but a pointer is 64bits big). + + + Since this issue can manifest itself in many ways (as there are many ways to + improperly truncate a pointer), you will need to read the source code + starting with the displayed warning. Make sure types are declared, used, + and passed properly. Make sure that all function prototypes are found (see + the Implicit Declarations + section for more information). So on and so forth. + + + diff --git a/pym/portage.py b/pym/portage.py index 1add16242..4f4dbf55e 100644 --- a/pym/portage.py +++ b/pym/portage.py @@ -2169,6 +2169,29 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, **keyw env=mysettings.environ() keywords["opt_name"]="[%s]" % mysettings["PF"] + # The default policy for the sesandbox domain only allows entry (via exec) + # from shells and from binaries that belong to portage (the number of entry + # points is minimized). The "tee" binary is not among the allowed entry + # points, so it is spawned outside of the sesandbox domain and reads from a + # pipe between two domains. + logfile = keywords.get("logfile") + mypids = [] + pw = None + if logfile: + del keywords["logfile"] + fd_pipes = keywords.get("fd_pipes") + if fd_pipes is None: + fd_pipes = {0:0, 1:1, 2:2} + elif 1 not in fd_pipes or 2 not in fd_pipes: + raise ValueError(fd_pipes) + pr, pw = os.pipe() + mypids.extend(portage_exec.spawn(('tee', '-i', '-a', logfile), + returnpid=True, fd_pipes={0:pr, 1:fd_pipes[1], 2:fd_pipes[2]})) + os.close(pr) + fd_pipes[1] = pw + fd_pipes[2] = pw + keywords["fd_pipes"] = fd_pipes + features = mysettings.features # XXX: Negative RESTRICT word droppriv=(droppriv and ("userpriv" in features) and not \ @@ -2194,12 +2217,33 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, **keyw con = con.replace(mysettings["PORTAGE_T"], mysettings["PORTAGE_SANDBOX_T"]) selinux.setexec(con) - retval = spawn_func(mystring, env=env, **keywords) - - if sesandbox: - selinux.setexec(None) - - return retval + returnpid = keywords.get("returnpid") + keywords["returnpid"] = True + try: + mypids.extend(spawn_func(mystring, env=env, **keywords)) + finally: + if pw: + os.close(pw) + if sesandbox: + selinux.setexec(None) + + if returnpid: + return mypids + + while mypids: + pid = mypids.pop(0) + retval = os.waitpid(pid, 0)[1] + portage_exec.spawned_pids.remove(pid) + if retval != os.EX_OK: + for pid in mypids: + if os.waitpid(pid, os.WNOHANG) == (0,0): + os.kill(pid, signal.SIGTERM) + os.waitpid(pid, 0) + portage_exec.spawned_pids.remove(pid) + if retval & 0xff: + return (retval & 0xff) << 8 + return retval >> 8 + return os.EX_OK def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks",use_locks=1, try_mirrors=1): "fetch files. Will use digest file if available." @@ -3897,6 +3941,8 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", def compare_pkgs(a, b): return pkgcmp(b[1], a[1]) portdb = trees[myroot]["porttree"].dbapi + if kwargs["use_binaries"]: + portdb = trees[myroot]["bintree"].dbapi myvirtuals = mysettings.getvirtuals() for x in mysplit: if x == "||": @@ -3919,11 +3965,19 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", match_atom = x if isblocker: match_atom = x[1:] - pkgs = [] + pkgs = {} for cpv in portdb.match(match_atom): # only use new-style matches if cpv.startswith("virtual/"): - pkgs.append((cpv, pkgsplit(cpv))) + pkgs[cpv] = (cpv, pkgsplit(cpv), portdb) + if kwargs["use_binaries"] and "vartree" in trees[myroot]: + vardb = trees[myroot]["vartree"].dbapi + for cpv in vardb.match(match_atom): + # only use new-style matches + if cpv.startswith("virtual/"): + if cpv in pkgs: + continue + pkgs[cpv] = (cpv, pkgsplit(cpv), vardb) if not (pkgs or mychoices): # This one couldn't be expanded as a new-style virtual. Old-style # virtuals have already been expanded by dep_virtual, so this one @@ -3935,13 +3989,14 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", if not pkgs and len(mychoices) == 1: newsplit.append(x.replace(mykey, mychoices[0])) continue + pkgs = pkgs.values() pkgs.sort(compare_pkgs) # Prefer higher versions. if isblocker: a = [] else: a = ['||'] for y in pkgs: - depstring = " ".join(portdb.aux_get(y[0], dep_keys)) + depstring = " ".join(y[2].aux_get(y[0], dep_keys)) if edebug: print "Virtual Parent: ", y[0] print "Virtual Depstring:", depstring @@ -4051,6 +4106,11 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): all_available = True for atom in atoms: if not mydbapi.match(atom): + # With --usepkgonly, count installed packages as "available". + # Note that --usepkgonly currently has no package.mask support. + # See bug #149816. + if use_binaries and vardb and vardb.match(atom): + continue all_available = False break diff --git a/pym/portage_news.py b/pym/portage_news.py index 50da11f8d..c06a726a0 100644 --- a/pym/portage_news.py +++ b/pym/portage_news.py @@ -7,7 +7,7 @@ from portage_const import INCREMENTALS, PROFILE_PATH, NEWS_LIB_PATH from portage import config, vartree, vardbapi, portdbapi from portage_util import ensure_dirs from portage_data import portage_gid -from portage_locks import lockfile, unlockfile +from portage_locks import lockfile, unlockfile, lockdir, unlockdir import os, re @@ -55,36 +55,48 @@ class NewsManager(object): raise ValueError("Invalid repoID: %s" % repoid) if os.path.exists(self.TIMESTAMP_PATH): + # Make sure the timestamp has correct permissions. + apply_permissions( unreadfile, 0, portage_gid, 664 ) timestamp = os.stat(self.TIMESTAMP_PATH).st_mtime else: timestamp = 0 path = os.path.join( self.portdb.getRepositoryPath( repoid ), self.NEWS_PATH ) - # Skip reading news for repoid if the news dir does not exist. Requested by - # NightMorph :) - if not os.path.exists( path ): - return None - news = os.listdir( path ) - updates = [] - for item in news: - try: - file = os.path.join( path, item, item + "." + self.LANGUAGE_ID + ".txt") - tmp = NewsItem( file , timestamp ) - except TypeError: - continue + try: + newsdir_lock = lockdir( self.portdb.getRepositoryPath ) + # Skip reading news for repoid if the news dir does not exist. Requested by + # NightMorph :) + if not os.path.exists( path ): + return None + news = os.listdir( path ) + updates = [] + for item in news: + try: + file = os.path.join( path, item, item + "." + self.LANGUAGE_ID + ".txt") + tmp = NewsItem( file , timestamp ) + except TypeError: + continue - if tmp.isRelevant( profile=os.readlink(PROFILE_PATH), config=config, vardb=self.vdb): - updates.append( tmp ) + if tmp.isRelevant( profile=os.readlink(PROFILE_PATH), config=config, vardb=self.vdb): + updates.append( tmp ) + finally: + unlockdir(newsdir_lock) + del path path = os.path.join( self.UNREAD_PATH, "news-" + repoid + ".unread" ) - unread_lock = lockfile( path ) - unread_file = open( path, "a" ) - for item in updates: - unread_file.write( item.path + "\n" ) + try: + unread_lock = lockfile( path ) + # Make sure we have the correct permissions when created + unread_file = open( path, "a" ) + apply_permissions( unreadfile, 0, portage_gid, 664 ) + + for item in updates: + unread_file.write( item.path + "\n" ) - unread_file.close() - unlockfile(unread_lock) + unread_file.close() + finally: + unlockfile(unread_lock) # Touch the timestamp file f = open(self.TIMESTAMP_PATH, "w") @@ -98,16 +110,25 @@ class NewsManager(object): check for new items. """ + unreadfile = os.path.join( self.UNREAD_PATH, "news-"+ repoid +".unread" ) + # Set correct permissions on the news-repoid.unread file + try: + apply_permissions( unreadfile, 0, portage_gid, 664 ) + except FileNotFound: + pass # It may not exist yet, thats ok. + if update: self.updateItems( repoid ) - - unreadfile = os.path.join( self.UNREAD_PATH, "news-"+ repoid +".unread" ) - unread_lock = lockfile(unreadfile) - if os.path.exists( unreadfile ): - unread = open( unreadfile ).readlines() - if len(unread): - return len(unread) - unlockfile(unread_lock) + + try: + unread_lock = lockfile(unreadfile) + if os.path.exists( unreadfile ): + unread = open( unreadfile ).readlines() + if len(unread): + return len(unread) + finally: + if unread_lock: + unlockfile(unread_lock) _installedRE = re.compile("Display-If-Installed:(.*)\n") _profileRE = re.compile("Display-If-Profile:(.*)\n")