From 0629e2a07a50f2e1474d749a9a5aef952ddaf60b Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Fri, 12 Jan 2007 22:30:39 +0000 Subject: [PATCH] Merged from trunk 5533:5549 For bug #161422, make slot collisions non-fatal in cases where an invalid depgraph is acceptable (--fetchonly, for example). Don't allow slot collision nodes to block other packages since blocker validation is only able to account for one package per slot. Rename depgraph._invalid_depgraph_is_acceptable() to _accept_collisions() for a better description. Add a bit of path mangling to allow deeper tests (WarnerBro) remove bogs test dir, add portage_versions (WarnerBro) edit test-notes (WarnerBro) (duh! you can do better) more test re-arranging as Alec can't read what functions are in what files (WarnerBro) Remove / character from module names. Remove RESTRICT=sandbox docs until bug #161045 is resolved (genone) Update CCACHE docs, patch by Ryan Hill (bug #156865) (genone) Add 'stricter' description, misc typo fixes, patch by Max Lorenz (bug #153755) (genone) Rename 'Last Sync' to 'Timestamp of tree' to clarify the meaning (bug #150051) (genone) Don't escape strings in PortageException as that breaks newline handling (bug 147785) (genone) More portage_util tests (WarnerBro) Extend ignore filter for grablines (bug #144445) (genone) add stack_lists checks (WarnerBro) svn path=/main/branches/prefix/; revision=5616 --- TEST-NOTES | 9 ++- bin/emerge | 66 +++++++++++-------- cnf/make.conf | 42 ++++++++---- man/ebuild.5 | 3 - man/make.conf.5 | 2 +- pym/portage_exception.py | 5 +- pym/portage_util.py | 4 +- tests/__init__.py | 9 +-- tests/portage_dep/__init__.py | 4 ++ .../test_match_from_list.py | 0 tests/portage_util/test_stackDicts.py | 32 ++++++++- tests/portage_util/test_stackLists.py | 13 +++- tests/portage_util/test_uniqueArray.py | 19 +++++- tests/portage_util/test_varExpand.py | 53 ++++++++++++++- tests/portage_versions/__init__.py | 4 ++ .../test_vercmp.py | 0 16 files changed, 203 insertions(+), 62 deletions(-) create mode 100644 tests/portage_dep/__init__.py rename tests/{portage => portage_dep}/test_match_from_list.py (100%) create mode 100644 tests/portage_versions/__init__.py rename tests/{portage => portage_versions}/test_vercmp.py (100%) diff --git a/TEST-NOTES b/TEST-NOTES index dbdbf20d0..82f342310 100644 --- a/TEST-NOTES +++ b/TEST-NOTES @@ -19,6 +19,9 @@ tests/portage_versions/test_vercmp.py would be the filename. -The __init__.py file in tests only goes 1 deep (it's non-recursive, in other words). If -you end up with more directories it will probably break at present. Need to replace the -'/' in the directory path with a '.' (for proper importing). This should be trivial to fix. +The __init__.py file now does recursive tests, but you need to tell it so. For example, if +you had cache tests the dir format would be something like... + +tests/cache/flat_hash/test_foo.py + +and you would put "cache/flat_hash" into the testDirs variable in __init__.py. diff --git a/bin/emerge b/bin/emerge index 20676976b..c4234c2b0 100755 --- a/bin/emerge +++ b/bin/emerge @@ -1013,6 +1013,9 @@ class depgraph: self.blocker_digraph = digraph() self.blocker_parents = {} self._slot_collision_info = [] + # Slot collision nodes are not allowed to block other packages since + # blocker validation is only able to account for one package per slot. + self._slot_collision_nodes = set() self._altlist_cache = {} self._pprovided_args = [] @@ -1096,7 +1099,8 @@ class depgraph: if mytype == "blocks": if myparent and \ "--buildpkgonly" not in self.myopts and \ - "--nodeps" not in self.myopts: + "--nodeps" not in self.myopts and \ + myparent not in self._slot_collision_nodes: mybigkey[1] = myparent[1] self.blocker_parents.setdefault( tuple(mybigkey), set()).add(myparent) @@ -1165,6 +1169,7 @@ class depgraph: slot_atom = "%s:%s" % (portage.dep_getkey(mykey), metadata["SLOT"]) existing_node = self._slot_node_map[myroot].get( slot_atom, None) + slot_collision = False if existing_node: e_type, myroot, e_cpv, e_status = existing_node if mykey == e_cpv: @@ -1189,31 +1194,27 @@ class depgraph: myparents.append(myparent) self._slot_collision_info.append( ((jbigkey, myparents), (existing_node, e_parents))) - - # Now add this node to the graph so that self.display() - # can show use flags and --tree output. This node is - # only being partially added to the graph. It must not be - # allowed to interfere with the other nodes that have been - # added. Do not overwrite data for existing nodes in - # self.pkg_node_map and self.mydbapi since that data will - # be used for blocker validation. - self.pkg_node_map[myroot].setdefault(mykey, jbigkey) - self.useFlags[myroot].setdefault(mykey, myuse) - self._parent_child_digraph.add(jbigkey, myparent) - if rev_dep and myparent: - self.digraph.add(myparent, jbigkey, - priority=priority) - else: - self.digraph.add(jbigkey, myparent, - priority=priority) - # The slot collision has rendered the graph invalid, so - # there's no need to process dependencies of this node. - return 1 - - self._slot_node_map[myroot][slot_atom] = jbigkey - self.pkg_node_map[myroot][mykey] = jbigkey - self.useFlags[myroot][mykey] = myuse - self.mydbapi[myroot].cpv_inject(mykey, metadata=metadata) + self._slot_collision_nodes.add(jbigkey) + slot_collision = True + + if slot_collision: + # Now add this node to the graph so that self.display() + # can show use flags and --tree output. This node is + # only being partially added to the graph. It must not be + # allowed to interfere with the other nodes that have been + # added. Do not overwrite data for existing nodes in + # self.pkg_node_map and self.mydbapi since that data will + # be used for blocker validation. + self.pkg_node_map[myroot].setdefault(mykey, jbigkey) + self.useFlags[myroot].setdefault(mykey, myuse) + # Even though the graph is now invalid, continue to process + # dependencies so that things like --fetchonly can still + # function despite collisions. + else: + self.mydbapi[myroot].cpv_inject(mykey, metadata=metadata) + self._slot_node_map[myroot][slot_atom] = jbigkey + self.pkg_node_map[myroot][mykey] = jbigkey + self.useFlags[myroot][mykey] = myuse if rev_dep and myparent: self.digraph.addnode(myparent, jbigkey, @@ -1939,9 +1940,18 @@ class depgraph: if x[0] == "blocks": return True self._show_slot_collision_notice(self._slot_collision_info[0]) - return False + if not self._accept_collisions(): + return False return True + def _accept_collisions(self): + acceptable = False + for x in ("--nodeps", "--pretend", "--fetchonly", "--fetch-all-uri"): + if x in self.myopts: + acceptable = True + break + return acceptable + def altlist(self, reversed=False): if reversed in self._altlist_cache: return self._altlist_cache[reversed][:] @@ -4210,7 +4220,7 @@ def action_info(settings, trees, myopts, myfiles): print "Unknown Host Operating System" lastSync = portage.grabfile(os.path.join( settings["PORTDIR"], "metadata", "timestamp.chk")) - print "Last Sync:", + print "Timestamp of tree:", if lastSync: print lastSync[0] else: diff --git a/cnf/make.conf b/cnf/make.conf index b3a85e7a6..ea913e36b 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -86,14 +86,14 @@ # # DISTDIR is where all of the source code tarballs will be placed for # emerges. The source code is maintained here unless you delete -# it. The entire repository of tarballs for gentoo is 9G. This is +# it. The entire repository of tarballs for Gentoo is 9G. This is # considerably more than any user will ever download. 2-3G is # a large DISTDIR. #DISTDIR=${PORTDIR}/distfiles # # PKGDIR is the location of binary packages that you can have created # with '--buildpkg' or '-b' while emerging a package. This can get -# upto several hundred megs, or even a few gigs. +# up to several hundred megs, or even a few gigs. #PKGDIR=${PORTDIR}/packages # # PORT_LOGDIR is the location where portage will store all the logs it @@ -248,13 +248,15 @@ # developers as well. # # 'assume-digests' -# when commiting work to cvs with repoman(1), assume that all +# when committing work to cvs with repoman(1), assume that all # existing SRC_URI digests are correct. This feature also # affects digest generation via ebuild(1) and emerge(1) (emerge # generates digests only when the 'digest' feature is enabled). # 'buildpkg' causes binary packages to be created of all packages that # are being merged. -# 'ccache' enables ccache support via CC. +# 'ccache' enable support for the dev-util/ccache package, which can +# noticably decrease the time needed to remerge previously built +# packages. # 'confcache' enable confcache support; speeds up autotool based configure # calls # 'collision-protect' @@ -282,7 +284,7 @@ # from a merge. # 'keepwork' prevents the clean phase from deleting the WORKDIR. # 'test' causes ebuilds to perform testing phases if they are capable -# of it. Some packages support this automaticaly via makefiles. +# of it. Some packages support this automatically via makefiles. # 'metadata-transfer' # automatically perform a metadata transfer when `emerge --sync` # is run. @@ -298,6 +300,9 @@ # 'sandbox' enables sandboxing when running emerge and ebuild. # 'strict' causes portage to react strongly to conditions that are # potentially dangerous, like missing/incorrect Manifest files. +# 'stricter' causes portage to react strongly to conditions that may +# conflict with system security provisions (for example +# textrels, executable stacks). # 'userfetch' when portage is run as root, drop privileges to # portage:portage during the fetching of package sources. # 'userpriv' allows portage to drop root privileges while it is compiling, @@ -306,16 +311,25 @@ # 'usersandbox' enables sandboxing while portage is running under userpriv. #FEATURES="sandbox buildpkg ccache distcc userpriv usersandbox notitles noclean noauto cvs keeptemp keepwork" #FEATURES="sandbox ccache distcc distlocks" + +# CCACHE_SIZE and CCACHE_DIR are used to control the behavior of ccache, and +# and are only used if "ccache" is in FEATURES. +# +# CCACHE_SIZE sets the space limitations for ccache. The default size is +# "2G", or 2 gigabytes. Units are specified with 'G', 'M', or 'K'. # -# CCACHE_SIZE sets the space use limitations for ccache. The default size is -# 2G, and will be set if not defined otherwise and ccache is in features. -# Portage will set the default ccache dir if it is not present in the -# user's environment, for userpriv it sets: ${PORTAGE_TMPDIR}/ccache -# (/var/tmp/ccache), and for regular use the default is /root/.ccache. -# Sizes are specified with 'G' 'M' or 'K'. -# '2G' for 2 gigabytes, '2048M' for 2048 megabytes (same as 2G). #CCACHE_SIZE="512M" # +# CCACHE_DIR sets the ccache path. If not specified, portage will default +# to "${PORTAGE_TMPDIR}/ccache". +# +# Note that to display ccache statistics outside of portage, you must +# remember to give the correct path to the cache. +# +# $ CCACHE_DIR=/var/tmp/ccache ccache -s +# +#CCACHE_DIR="${PORTAGE_TMPDIR}/ccache" + # DISTCC_DIR sets the temporary space used by distcc. #DISTCC_DIR="${PORTAGE_TMPDIR}/.distcc" @@ -366,7 +380,7 @@ PORTAGE_ELOG_CLASSES="warn error log" # Examples: #PORTAGE_ELOG_MAILURI="root@localhost localhost" (this is also the default setting) #PORTAGE_ELOG_MAILURI="user@some.domain mail.some.domain" (sends mails to user@some.domain using the mailserver mail.some.domain) -#PORTAGE_ELOG_MAILURI="user@some.domain user:secret@mail.some.domain:100465" (this is left uncommented as a reader excercise ;) +#PORTAGE_ELOG_MAILURI="user@some.domain user:secret@mail.some.domain:100465" (this is left uncommented as a reader exercise ;) # PORTAGE_ELOG_MAILFROM: you can set the from-address of logmails with this variable, # if unset mails are sent by "portage" (this default may fail @@ -374,7 +388,7 @@ PORTAGE_ELOG_CLASSES="warn error log" #PORTAGE_ELOG_MAILFROM="portage@some.domain" # PORTAGE_ELOG_MAILSUBJECT: template string to be used as subject for logmails. The following -# variables are exanded: +# variables are expanded: # ${PACKAGE} - see description of PORTAGE_ELOG_COMMAND # ${HOST} - FQDN of the host portage is running on #PORTAGE_ELOG_MAILSUBJECT="package \${PACKAGE} merged on \${HOST} with notice" diff --git a/man/ebuild.5 b/man/ebuild.5 index 209cd031b..cbcc0b9c3 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -341,9 +341,6 @@ fetch from URL's in \fBSRC_URI\fR before \fBGENTOO_MIRRORS\fR. .I strip final binaries/libraries will not be stripped of debug symbols. .TP -.I sandbox -disable sandbox (do not use without very good reason). -.TP .I test do not run src_test even if user has \fBFEATURES\fR=test. .TP diff --git a/man/make.conf.5 b/man/make.conf.5 index 035d4d58f..a7823cac6 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -133,7 +133,7 @@ Build binary packages for just packages in the system set. .B ccache Enable portage support for the ccache package. If the ccache dir is not present in the user's environment, then portage will default to -${PORTAGE_TMPDIR}/ccache (with \fIuserpriv\fR) and to /root/.ccache otherwise. +${PORTAGE_TMPDIR}/ccache. .TP .B confcache Enable confcache support; global configure caching, speeds up autotool based configure diff --git a/pym/portage_exception.py b/pym/portage_exception.py index 99dd473b0..0a9d8d130 100644 --- a/pym/portage_exception.py +++ b/pym/portage_exception.py @@ -8,7 +8,10 @@ class PortageException(Exception): def __init__(self,value): self.value = value[:] def __str__(self): - return repr(self.value) + if isinstance(self.value, str): + return self.value + else: + return repr(self.value) class CorruptionError(PortageException): """Corruption indication""" diff --git a/pym/portage_util.py b/pym/portage_util.py index 17e669d31..34c4d6588 100644 --- a/pym/portage_util.py +++ b/pym/portage_util.py @@ -228,10 +228,12 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0): def grablines(myfilename,recursive=0): mylines=[] if recursive and os.path.isdir(myfilename): + if myfilename in ["RCS", "CVS", "SCCS"]: + continue dirlist = os.listdir(myfilename) dirlist.sort() for f in dirlist: - if not f.startswith("."): + if not f.startswith(".") and not f.endswith("~"): mylines.extend(grablines( os.path.join(myfilename, f), recursive)) else: diff --git a/tests/__init__.py b/tests/__init__.py index 2808ed5a4..4ac94383f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -7,13 +7,13 @@ import os, unittest def main(): - testDirs = ["portage", "portage_util"] + testDirs = ["portage", "portage_util","portage_versions", "portage_dep"] suite = unittest.TestSuite() basedir = os.path.dirname(__file__) for mydir in testDirs: - suite.addTests(getTests(os.path.join(basedir, mydir))) + suite.addTests(getTests(os.path.join(basedir, mydir), basedir) ) return unittest.TextTestRunner(verbosity=2).run(suite) @@ -24,7 +24,7 @@ def my_import(name): mod = getattr(mod, comp) return mod -def getTests( path ): +def getTests( path, base_path ): """ path is the path to a given subdir ( 'portage/' for example) @@ -35,7 +35,8 @@ def getTests( path ): import os files = os.listdir( path ) files = [ f[:-3] for f in files if f.startswith("test_") and f.endswith(".py") ] - parent_module = ".".join(("tests", os.path.basename(path))) + parent_path = path[len(base_path)+1:] + parent_module = ".".join(("tests", parent_path)) result = [] for mymodule in files: try: diff --git a/tests/portage_dep/__init__.py b/tests/portage_dep/__init__.py new file mode 100644 index 000000000..e1c0ae595 --- /dev/null +++ b/tests/portage_dep/__init__.py @@ -0,0 +1,4 @@ +# tests/portage/__init__.py -- Portage Unit Test functionality +# Copyright 2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id: __init__.py 5522 2007-01-10 12:30:05Z antarus $ diff --git a/tests/portage/test_match_from_list.py b/tests/portage_dep/test_match_from_list.py similarity index 100% rename from tests/portage/test_match_from_list.py rename to tests/portage_dep/test_match_from_list.py diff --git a/tests/portage_util/test_stackDicts.py b/tests/portage_util/test_stackDicts.py index dd0743509..ac0e67ef8 100644 --- a/tests/portage_util/test_stackDicts.py +++ b/tests/portage_util/test_stackDicts.py @@ -4,8 +4,36 @@ # $Id: test_vercmp.py 5213 2006-12-08 00:12:41Z antarus $ from unittest import TestCase +from portage_util import stack_dicts + class StackDictsTestCase(TestCase): - def testStackDicts(self): - pass + def testStackDictsPass(self): + + tests = [ ( [ { "a":"b" }, { "b":"c" } ], { "a":"b", "b":"c" }, + False, [], False ), + ( [ { "a":"b" }, { "a":"c" } ], { "a":"b c" }, + True, [], False ), + ( [ { "a":"b" }, { "a":"c" } ], { "a":"b c" }, + False, ["a"], False ), + ( [ { "a":"b" }, None ], { "a":"b" }, + False, [], True ), + ( [ None ], None, False, [], False ), + ( [ None, {}], {}, False, [], True ) ] + + + for test in tests: + result = stack_dicts( test[0], test[2], test[3], test[4] ) + self.failIf( result != test[1], msg="Expected %s = %s" \ + % ( result, test[1] ) ) + + def testStackDictsFail(self): + + tests = [ ( [ None, {} ], None, False, [], True ), + ( [ { "a":"b"}, {"a":"c" } ], { "a":"b c" }, + False, [], False ) ] + for test in tests: + result = stack_dicts( test[0], test[2], test[3], test[4] ) + self.failIf( result == test[1], msg="Expected %s != %s, got \ + %s == %s!" % (result, test[1], result, test[1]) ) diff --git a/tests/portage_util/test_stackLists.py b/tests/portage_util/test_stackLists.py index 955f41ad8..c69139286 100644 --- a/tests/portage_util/test_stackLists.py +++ b/tests/portage_util/test_stackLists.py @@ -4,8 +4,19 @@ # $Id: test_vercmp.py 5213 2006-12-08 00:12:41Z antarus $ from unittest import TestCase +from portage_util import stack_lists class StackListsTestCase(TestCase): def testStackLists(self): - pass + + tests = [ ( [ ['a','b','c'], ['d','e','f'] ], ['a','c','b','e','d','f'], False ), + ( [ ['a','x'], ['b','x'] ], ['a','x','b'], False ), + ( [ ['a','b','c'], ['-*'] ], [], True ), + ( [ ['a'], ['-a'] ], [], True ) ] + + for test in tests: + result = stack_lists( test[0], test[2] ) + self.failIf( result != test[1], + msg="Got %s != %s from stack_lists( %s, %s )" \ + % ( result, test[1], test[0], test[2] ) ) diff --git a/tests/portage_util/test_uniqueArray.py b/tests/portage_util/test_uniqueArray.py index 654eb4b1c..bc0b750b9 100644 --- a/tests/portage_util/test_uniqueArray.py +++ b/tests/portage_util/test_uniqueArray.py @@ -4,8 +4,23 @@ # $Id: test_vercmp.py 5213 2006-12-08 00:12:41Z antarus $ from unittest import TestCase +from portage_util import unique_array class UniqueArrayTestCase(TestCase): - def testUniqueArray(self): - pass + def testUniqueArrayPass(self): + """ + test portage_util.uniqueArray() + """ + + import os + + tests = [ ( ["a","a","a",os,os,[],[],[]], ['a',os,[]] ), + ( [1,1,1,2,3,4,4] , [1,2,3,4]) ] + + for test in tests: + result = unique_array( test[0] ) + for item in test[1]: + number = result.count(item) + self.failIf( number is not 1, msg="%s contains %s of %s, \ + should be only 1" % (result, number, item) ) diff --git a/tests/portage_util/test_varExpand.py b/tests/portage_util/test_varExpand.py index e7bb62b67..f36027273 100644 --- a/tests/portage_util/test_varExpand.py +++ b/tests/portage_util/test_varExpand.py @@ -4,8 +4,57 @@ # $Id: test_vercmp.py 5213 2006-12-08 00:12:41Z antarus $ from unittest import TestCase, TestLoader +from portage_util import varexpand class VarExpandTestCase(TestCase): - def testVarexpand(self): - pass + def testVarExpandPass(self): + + varDict = { "a":"5", "b":"7", "c":"-5" } + for key in varDict.keys(): + result = varexpand( "$%s" % key, varDict ) + + self.failIf( result != varDict[key], + msg="Got %s != %s, from varexpand( %s, %s )" % \ + ( result, varDict[key], "$%s" % key, varDict ) ) + result = varexpand( "${%s}" % key, varDict ) + self.failIf( result != varDict[key], + msg="Got %s != %s, from varexpand( %s, %s )" % \ + ( result, varDict[key], "${%s}" % key, varDict ) ) + + def testVarExpandDoubleQuotes(self): + + varDict = { "a":"5" } + tests = [ ("\"${a}\"", "5") ] + for test in tests: + result = varexpand( test[0], varDict ) + self.failIf( result != test[1], + msg="Got %s != %s from varexpand( %s, %s )" \ + % ( result, test[1], test[0], varDict ) ) + + def testVarExpandSingleQuotes(self): + + varDict = { "a":"5" } + tests = [ ("\'${a}\'", "${a}") ] + for test in tests: + result = varexpand( test[0], varDict ) + self.failIf( result != test[1], + msg="Got %s != %s from varexpand( %s, %s )" \ + % ( result, test[1], test[0], varDict ) ) + + def testVarExpandFail(self): + + varDict = { "a":"5", "b":"7", "c":"15" } + + testVars = [ "fail" ] + + for var in testVars: + result = varexpand( "$%s" % var, varDict ) + self.failIf( len(result), + msg="Got %s == %s, from varexpand( %s, %s )" \ + % ( result, var, "$%s" % var, varDict ) ) + + result = varexpand( "${%s}" % var, varDict ) + self.failIf( len(result), + msg="Got %s == %s, from varexpand( %s, %s )" \ + % ( result, var, "${%s}" % var, varDict ) ) diff --git a/tests/portage_versions/__init__.py b/tests/portage_versions/__init__.py new file mode 100644 index 000000000..e1c0ae595 --- /dev/null +++ b/tests/portage_versions/__init__.py @@ -0,0 +1,4 @@ +# tests/portage/__init__.py -- Portage Unit Test functionality +# Copyright 2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id: __init__.py 5522 2007-01-10 12:30:05Z antarus $ diff --git a/tests/portage/test_vercmp.py b/tests/portage_versions/test_vercmp.py similarity index 100% rename from tests/portage/test_vercmp.py rename to tests/portage_versions/test_vercmp.py -- 2.26.2