From 04526da5ae8e4d01fcef25d373bf64b928c33fd0 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Wed, 28 Mar 2007 19:20:08 +0000 Subject: [PATCH] Merged from trunk 6278:6291 +------------------------------------------------------------------------+ | 6279 | Fix the rest of the tests | | WarnerBro | | |-----------+------------------------------------------------------------| | 6280 | completely rewrite file bits, use Loader classes to push | | WarnerBro | loading functionality down | |-----------+------------------------------------------------------------| | 6281 | split the loader classes into their own file, split the | | WarnerBro | recursive filename grabbing out into it's own function | |-----------+------------------------------------------------------------| | 6282 | Do the "*" check as early as possible in | | zmedico | config.getMissingLicenses(). | |-----------+------------------------------------------------------------| | 6283 | Verify the Manifest first even if there's only one package | | zmedico | (prior to the fetch phase). | |-----------+------------------------------------------------------------| | 6285 | fix grammar in error output | | SpankMan | | |-----------+------------------------------------------------------------| | 6286 | Fix insecure tempfile usage | | WarnerBro | | |-----------+------------------------------------------------------------| | 6287 | fix some tests, break others | | WarnerBro | | |-----------+------------------------------------------------------------| | 6288 | clean up some init functions, add PortageModules class | | WarnerBro | | |-----------+------------------------------------------------------------| | 6289 | fix tests for PortageModulesFile | | WarnerBro | | |-----------+------------------------------------------------------------| | 6290 | fix broken PortageModule classes | | WarnerBro | | |-----------+------------------------------------------------------------| | 6291 | minor name changes for bin tests, svn propset keywords Id | | WarnerBro | stuff | +------------------------------------------------------------------------+ svn path=/main/branches/prefix/; revision=6312 --- pym/emerge/__init__.py | 1 - pym/portage/__init__.py | 9 +- pym/portage/env/config.py | 143 ++++++++------- pym/portage/env/loaders.py | 166 ++++++++++++++++++ pym/portage/tests/bin/test_dobin.py | 2 +- pym/portage/tests/bin/test_dodir.py | 4 +- .../env/config/test_PackageKeywordsFile.py | 19 +- .../tests/env/config/test_PackageMaskFile.py | 8 +- .../tests/env/config/test_PackageUseFile.py | 12 +- .../env/config/test_PortageModulesFile.py | 34 ++++ 10 files changed, 301 insertions(+), 97 deletions(-) create mode 100644 pym/portage/env/loaders.py create mode 100644 pym/portage/tests/env/config/test_PortageModulesFile.py diff --git a/pym/emerge/__init__.py b/pym/emerge/__init__.py index 7b58c116e..728b6b984 100644 --- a/pym/emerge/__init__.py +++ b/pym/emerge/__init__.py @@ -2938,7 +2938,6 @@ class MergeTask(object): # as soon as possible. if "--fetchonly" not in self.myopts and \ "--fetch-all-uri" not in self.myopts and \ - len(mylist) > 1 and \ "strict" in self.settings.features: shown_verifying_msg = False quiet_settings = {} diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 13a335c49..aa8d2a24d 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1774,13 +1774,14 @@ class config: @rtype: List @return: A list of licenses that have not been accepted. """ + if "*" in self._accept_license: + return [] + acceptable_licenses = self._accept_license cpdict = self._plicensedict.get(dep_getkey(cpv), None) - acceptable_licenses = self._accept_license.copy() if cpdict: + acceptable_licenses = self._accept_license.copy() for atom in match_to_list(cpv, cpdict.keys()): acceptable_licenses.update(cpdict[atom]) - if "*" in acceptable_licenses: - return [] license_struct = portage.dep.paren_reduce(licenses) license_struct = portage.dep.use_reduce( license_struct, uselist=uselist) @@ -3795,7 +3796,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # forget it and are using ebuild if not os.path.exists( os.path.join(mysettings["PORTAGE_BUILDDIR"], ".installed")): - writemsg("!!! mydo=qmerge, but install phase hasn't been ran\n", + writemsg("!!! mydo=qmerge, but the install phase has not been run\n", noiselevel=-1) return 1 # qmerge is a special phase that implies noclean. diff --git a/pym/portage/env/config.py b/pym/portage/env/config.py index 675e01057..3bac9a3a5 100644 --- a/pym/portage/env/config.py +++ b/pym/portage/env/config.py @@ -3,8 +3,8 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ -import os from UserDict import UserDict +from portage.env.loaders import KeyListFileLoader, KeyValuePairFileLoader, AtomFileLoader class PackageKeywords(UserDict): """ @@ -16,6 +16,12 @@ class PackageKeywords(UserDict): data = {} + def __init__(self, loader): + self._loader = loader + + def load(self): + self.data, self.errors = self._loader.load() + def iteritems(self): return self.data.iteritems() @@ -32,33 +38,12 @@ class PackageKeywordsFile(PackageKeywords): """ Inherits from PackageKeywords; implements a file-based backend. Doesn't handle recursion yet. """ - def __init__( self, filename ): - self.fname = filename - - def load(self, recursive): - """ - Package.keywords files have comments that begin with #. - The entries are of the form: - >>> cpv [-~]keyword1 [-~]keyword2 keyword3 - >>> Exceptions include -*, ~*, and ** for keywords. - """ - - if os.path.exists( self.fname ): - f = open(self.fname, 'rb') - for line in f: - if line.startswith('#'): - continue - split = line.split() - if len(split): - # Surprisingly this works for iterables of length 1 - # fex ['sys-apps/portage','x86','amd64'] becomes {'sys-apps/portage':['x86','amd64']} - key, items = split[0],split[1:] - # if they specify the same cpv twice; stack the values (append) instead of overwriting. - if key in self.data: - self.data[key].append(items) - else: - self.data[key] = items + default_loader = KeyListFileLoader + + def __init__(self, filename): + PackageKeywords.__init__(self, self.default_loader(filename)) + class PackageUse(UserDict): """ A base class stub for things to inherit from; some people may want a database based package.keywords or something @@ -69,6 +54,12 @@ class PackageUse(UserDict): data = {} + def __init__(self, loader): + self._loader = loader + + def load( self): + self.data, self.errors = self._loader.load() + def iteritems(self): return self.data.iteritems() @@ -85,40 +76,23 @@ class PackageUseFile(PackageUse): """ Inherits from PackageUse; implements a file-based backend. Doesn't handle recursion yet. """ - def __init__(self, filename): - self.fname = filename - - def load(self, recursive): - """ - Package.keywords files have comments that begin with #. - The entries are of the form: - >>> atom useflag1 useflag2 useflag3.. - useflags may optionally be negative with a minus sign (-) - >>> atom useflag1 -useflag2 useflag3 - """ - - if os.path.exists( self.fname ): - f = open(self.fname, 'rb') - for line in f: - if line.startswith('#'): - continue - split = line.split() - if len(split): - # Surprisingly this works for iterables of length 1 - # fex ['sys-apps/portage','foo','bar'] becomes {'sys-apps/portage':['foo','bar']} - key, items = split[0],split[1:] - # if they specify the same cpv twice; stack the values (append) instead of overwriting. - if key in self.data: - self.data[key].append(items) - else: - self.data[key] = items + default_loader = KeyListFileLoader + def __init__(self, filename): + PackageUse.__init__(self, self.default_loader(filename)) + class PackageMask(UserDict): """ A base class for Package.mask functionality """ data = {} + def __init__(self, loader): + self._loader = loader + + def load(self): + self.data, self.errors = self._loader.load() + def iteritems(self): return self.data.iteritems() @@ -146,24 +120,45 @@ class PackageMaskFile(PackageMask): to revert a previous mask; this only works when masking files are stacked """ + default_loader = AtomFileLoader + def __init__(self, filename): - self.fname = filename - - def load(self, recursive): - """ - Package.keywords files have comments that begin with #. - The entries are of the form: - >>> atom useflag1 useflag2 useflag3.. - useflags may optionally be negative with a minus sign (-) - >>> atom useflag1 -useflag2 useflag3 - """ - - if os.path.exists( self.fname ): - f = open(self.fname, 'rb') - for line in f: - if line.startswith('#'): - continue - split = line.split() - if len(split): - atom = split[0] # This is an extra assignment, but I think it makes the code more explicit in what goes into the dict - self.data[atom] = None # we only care about keys in the dict, basically abusing it as a list + PackageMask.__init__(self, self.default_loader(filename)) + +class PortageModules(UserDict): + """ + Base Class for user level module over-rides + """ + + data = {} + + def __init__(self, loader): + self._loader = loader + + def load(self): + self.data, self.errors = self._loader.load() + + def iteritems(self): + return self.data.iteritems() + + def __hash__(self): + return self.data.__hash__() + + def __contains__(self, key): + return key in self.data + + def keys(self): + return self.data.keys() + + def iterkeys(self): + return self.data.iterkeys() + +class PortageModulesFile(PortageModules): + """ + File Class for /etc/portage/modules + """ + + default_loader = KeyValuePairFileLoader + + def __init__(self, filename): + PortageModules.__init__(self, self.default_loader(filename)) diff --git a/pym/portage/env/loaders.py b/pym/portage/env/loaders.py new file mode 100644 index 000000000..c19c4001e --- /dev/null +++ b/pym/portage/env/loaders.py @@ -0,0 +1,166 @@ +# config.py -- Portage Config +# Copyright 2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +import os + +def RecursiveFileLoader(filename): + """ + If filename is of type file, return [filename] + else if filename is of type directory, return an array + full of files in that directory to process. + + Ignore files beginning with . or ending in ~. + Prune CVS directories. + + @param filename: name of a file/directory to traverse + @rtype: list + @returns: List of files to process + """ + + if os.path.isdir(filename): + for root, dirs, files in os.walk(self.fname): + if 'CVS' in dirs: + dirs.remove('CVS') + files = filter(files,startswith('.')) + files = filter(files,endswith('~')) + for file in files: + yield file + else: + yield filename + +class DataLoader(object): + + def load(self): + """ + Function to do the actual work of a Loader + """ + pass + +class AtomFileLoader(DataLoader): + """ + Class to load data from a file full of atoms one per line + + >>> atom1 + >>> atom2 + >>> atom3 + + becomes ['atom1', 'atom2', 'atom3'] + """ + + _recursive = False + + def __init__(self, filename): + DataLoader.__init__(self) + self.fname = filename + + def load(self): + data = {} + errors = {} + line_count = 0 + for file in RecursiveFileLoader(self.fname): + f = open(file, 'rb') + for line in f: + line_count = line_count + 1 + if line.startswith('#'): + continue + split = line.strip().split() + if not len(split): + errors.setdefault(self.fname,[]).append( + "Malformed data at line: %s, data: %s" + % (line_count, split)) + key = split[0] + data[key] = None + return (data,errors) + +class KeyListFileLoader(DataLoader): + """ + Class to load data from a file full of key [list] tuples + + >>>>key foo1 foo2 foo3 + becomes + {'key':['foo1','foo2','foo3']} + """ + + _recursive = False + + def __init__(self, filename): + DataLoader.__init__(self) + self.fname = filename + + def load(self): + data = {} + errors = {} + line_count = 0 + for file in RecursiveFileLoader(self.fname): + f = open(file, 'rb') + for line in f: + line_count = line_count + 1 + if line.startswith('#'): + continue + split = line.strip().split() + if len(split) < 2: + errors.setdefault(self.fname,[]).append( + "Malformed data at line: %s, data: %s" + % (line_count, split)) + key = split[0] + value = split[1:] + if key in data: + data[key].append(value) + else: + data[key] = value + return (data,errors) + +class KeyValuePairFileLoader(DataLoader): + """ + Class to load data from a file full of key=value pairs + + >>>>key=value + >>>>foo=bar + becomes: + {'key':'value', + 'foo':'bar'} + """ + + _recursive = False + + def __init__(self, filename): + DataLoader.__init__(self) + self.fname = filename + + def load(self): + """ + Return the {source: {key: value}} pairs from a file + Return the {source: [list of errors] from a load + + @param recursive: If set and self.fname is a directory; + load all files in self.fname + @type: Boolean + @rtype: tuple + @returns: + Returns (data,errors), both may be empty dicts or populated. + """ + + DataLoader.load(self) + data = {} + errors = {} + line_count = 0 + for file in RecursiveFileLoader(self.fname): + f = open(file, 'rb') + for line in f: + line_count = line_count + 1 # Increment line count + if line.startswith('#'): + continue + split = line.strip().split('=') + if len(split) < 2: + errors.setdefault(self.fname,[]).append( + "Malformed data at line: %s, data %s" + % (line_count, split)) + key = split[0] + value = split[1:] + if key in data: + data[key].append(value) + else: + data[key] = value + return (data,errors) diff --git a/pym/portage/tests/bin/test_dobin.py b/pym/portage/tests/bin/test_dobin.py index 074c8944e..b6ca12c72 100644 --- a/pym/portage/tests/bin/test_dobin.py +++ b/pym/portage/tests/bin/test_dobin.py @@ -6,7 +6,7 @@ from setup_env import * class DoBin(BinTestCase): - def testBasic(self): + def testDoBin(self): dobin("does-not-exist", 1) xexists_in_D("does-not-exist") xexists_in_D("/bin/does-not-exist") diff --git a/pym/portage/tests/bin/test_dodir.py b/pym/portage/tests/bin/test_dodir.py index a6ec5e196..988b67b8d 100644 --- a/pym/portage/tests/bin/test_dodir.py +++ b/pym/portage/tests/bin/test_dodir.py @@ -1,12 +1,12 @@ # test_dodir.py -- Portage Unit Testing Functionality # Copyright 2007 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -# $Id: test_dep_getcpv.py 6182 2007-03-06 07:35:22Z antarus $ +# $Id: test_dodir.py 6182 2007-03-06 07:35:22Z antarus $ from setup_env import * class DoDir(BinTestCase): - def testBasic(self): + def testDoDir(self): dodir("usr /usr") exists_in_D("/usr") dodir("/var/lib/moocow") diff --git a/pym/portage/tests/env/config/test_PackageKeywordsFile.py b/pym/portage/tests/env/config/test_PackageKeywordsFile.py index 32e9c30c5..fb053153b 100644 --- a/pym/portage/tests/env/config/test_PackageKeywordsFile.py +++ b/pym/portage/tests/env/config/test_PackageKeywordsFile.py @@ -5,14 +5,15 @@ from portage.tests import TestCase from portage.env.config import PackageKeywordsFile +from tempfile import mkstemp +import os class PackageKeywordsFileTestCase(TestCase): - fname = 'package.keywords' - cpv = 'sys-apps/portage' + cpv = ['sys-apps/portage'] keywords = ['~x86', 'amd64', '-mips'] - def testPackageKeywordsLoad(self): + def testPackageKeywordsFile(self): """ A simple test to ensure the load works properly """ @@ -20,16 +21,20 @@ class PackageKeywordsFileTestCase(TestCase): self.BuildFile() try: f = PackageKeywordsFile(self.fname) - f.load(recursive=False) + f.load() + i = 0 for cpv, keyword in f.iteritems(): - self.assertEqual( cpv, self.cpv ) + self.assertEqual( cpv, self.cpv[i] ) [k for k in keyword if self.assertTrue(k in self.keywords)] + i = i + 1 finally: self.NukeFile() def BuildFile(self): - f = open(self.fname, 'wb') - f.write('%s %s\n' % (self.cpv, ' '.join(self.keywords))) + fd, self.fname = mkstemp() + f = os.fdopen(fd, 'w') + for c in self.cpv: + f.write("%s %s\n" % (c,' '.join(self.keywords))) f.close() def NukeFile(self): diff --git a/pym/portage/tests/env/config/test_PackageMaskFile.py b/pym/portage/tests/env/config/test_PackageMaskFile.py index 83c9095fd..ad87d5a56 100644 --- a/pym/portage/tests/env/config/test_PackageMaskFile.py +++ b/pym/portage/tests/env/config/test_PackageMaskFile.py @@ -7,22 +7,22 @@ import os from portage.env.config import PackageMaskFile from portage.tests import TestCase +from tempfile import mkstemp class PackageMaskFileTestCase(TestCase): atoms = ['sys-apps/portage','dev-util/diffball','not@va1id@t0m'] - def testPackageMaskLoad(self): + def testPackageMaskFile(self): self.BuildFile() try: f = PackageMaskFile(self.fname) - f.load(recursive=False) - [atom for atom in f.iterkeys() if self.assertTrue(atom in self.atoms)] + f.load() + [atom for atom in f.keys() if self.assertTrue(atom in self.atoms)] finally: self.NukeFile() def BuildFile(self): - from tempfile import mkstemp fd, self.fname = mkstemp() f = os.fdopen(fd, 'w') f.write("\n".join(self.atoms)) diff --git a/pym/portage/tests/env/config/test_PackageUseFile.py b/pym/portage/tests/env/config/test_PackageUseFile.py index 4638923df..93474b6f7 100644 --- a/pym/portage/tests/env/config/test_PackageUseFile.py +++ b/pym/portage/tests/env/config/test_PackageUseFile.py @@ -3,23 +3,25 @@ # Distributed under the terms of the GNU General Public License v2 # $Id: test_PackageUseFile.py 6182 2007-03-06 07:35:22Z antarus $ +import os from portage.tests import TestCase from portage.env.config import PackageUseFile +from tempfile import mkstemp + class PackageUseFileTestCase(TestCase): - fname = 'package.use' cpv = 'sys-apps/portage' useflags = ['cdrom', 'far', 'boo', 'flag', 'blat'] - def testPackageUseLoad(self): + def testPackageUseFile(self): """ A simple test to ensure the load works properly """ self.BuildFile() try: f = PackageUseFile(self.fname) - f.load(recursive=False) + f.load() for cpv, use in f.iteritems(): self.assertEqual( cpv, self.cpv ) [flag for flag in use if self.assertTrue(flag in self.useflags)] @@ -27,8 +29,10 @@ class PackageUseFileTestCase(TestCase): self.NukeFile() def BuildFile(self): - f = open(self.fname, 'wb') + fd, self.fname = mkstemp() + f = os.fdopen(fd, 'wb') f.write("%s %s" % (self.cpv, ' '.join(self.useflags))) + f.close() def NukeFile(self): import os diff --git a/pym/portage/tests/env/config/test_PortageModulesFile.py b/pym/portage/tests/env/config/test_PortageModulesFile.py new file mode 100644 index 000000000..b523cbe7b --- /dev/null +++ b/pym/portage/tests/env/config/test_PortageModulesFile.py @@ -0,0 +1,34 @@ +import os + +from portage.tests import TestCase +from portage.env.config import PortageModulesFile +from tempfile import mkstemp +from itertools import izip + +class PortageModulesFileTestCase(TestCase): + + keys = ['foo.bar','baz','bob','extra_key'] + modules = ['spanky','zmedico','antarus','ricer'] + + def setUp(self): + self.items = {} + for k,v in izip(self.keys, self.modules): + self.items[k] = v + + def testPortageModulesFile(self): + self.BuildFile() + f = PortageModulesFile(self.fname) + for k in f.keys(): + self.assertEqual( f[k], self.items[k] ) + self.NukeFile() + + def BuildFile(self): + fd, self.fname = mkstemp() + f = os.fdopen(fd, 'wb') + for k,v in self.items.iteritems(): + f.write('%s=%s\n' % (k,v)) + f.close() + + def NukeFile(self): + import os + os.unlink(self.fname) -- 2.26.2