From 6abdc8a5021ad9ca2cadeafd80c69402ae509c37 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 7 Jun 2011 09:48:21 -0400 Subject: [PATCH] Rework config to use ConfigParser. This removes the ConfigObj dependency. Also: * reworked logging so you can override the default using logging.config.fileConfig. * remove internal sys.exit calls and rely on bubbling exceptions. --- g_pypi/__init__.py | 19 +++++ g_pypi/cli.py | 95 ++++++++++++++++--------- g_pypi/config.py | 136 ++++++++++++++--------------------- g_pypi/ebuild.py | 154 +++++++++++++++++++--------------------- g_pypi/portage_utils.py | 25 ++++--- scripts/g-pypi | 7 +- 6 files changed, 229 insertions(+), 207 deletions(-) diff --git a/g_pypi/__init__.py b/g_pypi/__init__.py index d3ed7e1..4f665ec 100644 --- a/g_pypi/__init__.py +++ b/g_pypi/__init__.py @@ -7,5 +7,24 @@ g-pypi creates Gentoo ebuilds by querying The Python Package Index """ +import logging + + __docformat__ = 'restructuredtext' __version__ = '0.2.2' + + +def _setup_default_logger(): + logger = logging.getLogger('g-pypi') + if logger.handlers: + return # logger already initialized + #logger.setLevel(logging.WARNING) + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler() + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + +_setup_default_logger() diff --git a/g_pypi/cli.py b/g_pypi/cli.py index 38c06d8..8bc61ee 100755 --- a/g_pypi/cli.py +++ b/g_pypi/cli.py @@ -12,6 +12,7 @@ Command-line code for g-pypi """ +import logging import sys import optparse import inspect @@ -27,7 +28,7 @@ except ImportError: from yolk.pypi import CheeseShop from yolk.yolklib import get_highest_version from yolk.setuptools_support import get_download_uri -from g_pypi.config import MyConfig +from g_pypi.config import CONFIG from g_pypi.ebuild import Ebuild from g_pypi.portage_utils import ebuild_exists from g_pypi.__init__ import __version__ as VERSION @@ -84,7 +85,7 @@ class GPyPI(object): Main class for command-line interface """ - def __init__(self, package_name, version, options, logger): + def __init__(self, package_name, version, config, logger): """ @param package_name: case-insensitive package name @type package_name: string @@ -101,7 +102,7 @@ class GPyPI(object): self.package_name = package_name self.version = version - self. options = options + self.config = config self.logger = logger self.tree = [(package_name, version)] self.pypi = CheeseShop() @@ -118,8 +119,8 @@ class GPyPI(object): #XXX: Call function to do 'ebuild pkg-ver.ebuild clean' etc. #to clean up unpacked ebuilds - self.logger.error("Error: " + msg) - sys.exit(1) + self.logger.error(msg) + raise Exception(msg) def create_ebuilds(self): """ @@ -129,6 +130,7 @@ class GPyPI(object): #Create first ebuild then turn off overwrite in case a dependency #ebuild already exists #self.logger.debug("Creating dep tree...") + no_deps = self.config.getboolean('options', 'no_deps') while len(self.tree): (project_name, version) = self.tree.pop(0) #self.logger.debug(self.tree) @@ -139,14 +141,14 @@ class GPyPI(object): #print "REQUIRES", requires if requires: for req in requires: - if self.options.no_deps or ebuild_exists("dev-python/%s" % req.project_name.lower()): - if not self.options.no_deps: + if no_deps or ebuild_exists("dev-python/%s" % req.project_name.lower()): + if not no_deps: self.logger.info("Skipping dependency (exists): %s" % req.project_name) else: self.add_dep(req.project_name) #Only force overwriting and category on first ebuild created, not dependencies - self.options.overwrite = False - self.options.category = None + self.config.setboolean('options', 'overwrite', False) + self.config.set('options', 'category', '') def add_dep(self, project_name): '''Add dependency''' @@ -217,7 +219,7 @@ class GPyPI(object): except ValueError: self.raise_error("The package seems to have a ridiculous name or version, can't proceed.") - if self.options.subversion: + if self.config.getboolean('options', 'live'): src_uri = get_download_uri(self.package_name, "dev", "source") else: src_uri = get_download_uri(self.package_name, self.version, "source") @@ -250,12 +252,11 @@ class GPyPI(object): else: self.version = get_highest_version(versions) - if self.options.uri: - download_url = self.options.uri - else: - download_url = self.get_uri() + uri = self.config.get('options', 'uri') + if not uri: + uri = self.get_uri() try: - ebuild = Ebuild(self.package_name, self.version, download_url) + ebuild = Ebuild(self.package_name, self.version, uri) except portage_exception.InvalidVersionString: self.logger.error("Can't determine PV, use -v to set it: %s-%s" % \ (self.package_name, self.version)) @@ -264,11 +265,13 @@ class GPyPI(object): self.logger.error("Can't determine PN, use -n to set it: %s-%s" % \ (self.package_name, self.version)) return + self.logger.debug('initialized Ebuild instance') ebuild.set_metadata(self.query_metadata()) ebuild.get_ebuild() - if self.options.pretend: + + if self.config.getboolean('options', 'pretend'): print ebuild.print_ebuild() return @@ -319,6 +322,8 @@ def show_version(): def main(): """Parse command-line options and do it.""" + logger = logging.getLogger('g-pypi') + usage = "usage: %prog [options] " opt_parser = optparse.OptionParser(usage=usage) @@ -363,16 +368,22 @@ def main(): "uri", default=False, help= "Specify URI of package if PyPI doesn't have it.") - opt_parser.add_option("-l", "--overlay", action='store', dest= - 'overlay', metavar='OVERLAY_NAME', default=None, help= - 'Specify overy to use by name ($OVERLAY/profiles/repo_name)') + opt_parser.add_option( + '-y', '--overlay', action='store', dest='overlay', + metavar='OVERLAY_NAME', default=None, + help='Specify overy to use by name ($OVERLAY/profiles/repo_name)') + + opt_parser.add_option( + '-Y', '--overlay-path', action='store', dest='overlay_path', + metavar='OVERLAY_PATH', default=None, + help='Specify overy to use by path (overrides `--ovelay`)') opt_parser.add_option("--format", action='store', dest= "format", default=None, help= "Format when printing to stdout: ansi, html, bbcode, or none") - opt_parser.add_option("-s", "--subversion", action='store_true', dest= - "subversion", default=False, help= - "Create live subversion ebuild if repo is available.") + opt_parser.add_option( + '-l', '--live', action='store_true', dest='live', default=False, + help='Create live ebuild from version control if repo is available.') opt_parser.add_option("-V", "--verbose", action='store_true', dest= "verbose", default=False, help= @@ -399,18 +410,38 @@ def main(): sys.stdout = StdOut(sys.stdout, ['distutils.log']) sys.stderr = StdOut(sys.stderr, ['distutils.log']) - config = MyConfig() - config.set_options(options) - config.set_logger() - logger = config.get_logger() - + for option in opt_parser.option_list: + if option.action in ['store_true', 'store_false']: + _set = CONFIG.setboolean + elif option.action == 'store': + _set = CONFIG.set + elif option.action == 'help': + continue + else: + logger.error('no action-to-type conversion for %s' % option.action) + raise NotImplementedError() + option_name = option.dest + section = 'core' + if not CONFIG.has_option(section, option_name): + section = 'options' + value = getattr(options, option.dest) + if value is not None: # override default with a user-specified option + _set(section, option_name, value) + if not package_spec: opt_parser.print_help() - logger.error("\nError: You need to specify a package name at least.") - return 1 + logger.error('you need to specify a package name') + raise ValueError(package_spec) (package_name, version) = parse_pkg_ver(package_spec) - gpypi = GPyPI(package_name, version, options, logger) + gpypi = GPyPI(package_name, version, CONFIG, logger) -if __name__ == "__main__": - sys.exit(main()) +if __name__ == "__main__": + logger = logging.getLogger('g-pypi') + try: + main() + except Exception, e: + logger.error(e) + raise # for debugging + sys.exit(1) + sys.exit(0) diff --git a/g_pypi/config.py b/g_pypi/config.py index 9593d25..6c4beda 100644 --- a/g_pypi/config.py +++ b/g_pypi/config.py @@ -1,101 +1,73 @@ #!/usr/bin/env python -# pylint: disable-msg=R0201 -# method could be function but we need shared class data - """ config.py ========= -Creates and reads config file using ConfigObj - - config['keyword'] = get_keyword() - config['overlay'] = get_portdir_overlay() - config['format'] = "ansi" +Creates and reads config file using ConfigParser:: + [core] + keyword = x86 + overlay path = ~/src/my-overlay/ + format = ansi """ +from ConfigParser import RawConfigParser as _Parser +from ConfigParser import NoSectionError, NoOptionError import os import logging - -from configobj import ConfigObj +from logging.config import fileConfig from g_pypi.portage_utils import get_keyword, get_portdir_overlay + __docformat__ = 'restructuredtext' CONFIG_DIR = os.path.expanduser("~/.g-pypi") - -class MyConfig: - - """ - Holds options from config file - """ - - config = None - options = None - logger = None - - def __init__(self): - self.set_config(self.get_config()) - - def get_config(self): - """Read config file, create if it doesn't exist""" - if not os.path.exists(self.get_rc_filename()): - self.create_config() - return ConfigObj(self.get_rc_filename()) - - def create_config(self): - """Create config file with defaults""" - if not os.path.exists(CONFIG_DIR): - os.mkdir(CONFIG_DIR) - self.create_config_obj() - - def create_config_obj(self): - """Set defaults for ConigObj""" - config = ConfigObj() - config.filename = self.get_rc_filename() - config['keyword'] = get_keyword() - config['overlay'] = get_portdir_overlay() - config['format'] = "ansi" - config['background'] = "dark" - config.write() - self.set_config(config) - #logger isn't set yet - print "Your default keyword will be: %s " % \ - config['keyword'] - print "Your default overlay will be: %s " % \ - config['overlay'] - print "To change these edit: %s \n\n" % config.filename - - def set_config(self, config): - """Set config""" - MyConfig.config = config - - def set_options(self, options): - """Set options""" - MyConfig.options = options - - def get_rc_filename(self): - """Return rc_file filename""" - return os.path.join(CONFIG_DIR, "g-pypirc") - - def set_logger(self): - """Set logger""" - MyConfig.logger = logging.getLogger("g-pypi") - if MyConfig.options.verbose: - MyConfig.logger.setLevel(logging.INFO) - elif MyConfig.options.quiet: - MyConfig.logger.setLevel(logging.ERROR) - elif MyConfig.options.debug: - MyConfig.logger.setLevel(logging.DEBUG) - else: - MyConfig.logger.setLevel(logging.INFO) - MyConfig.logger.addHandler(logging.StreamHandler()) - - def get_logger(self): - """Return logging object""" - return MyConfig.logger - +def get_rc_filename(): + """Return rc_file filename""" + return os.path.join(CONFIG_DIR, "g-pypirc") + +def setup_logger(): + "Possibly overrides 'g-pypi' logger stored in logging package" + logger = logging.getLogger('g-pypi') + try: + fileConfig(fname=get_rc_filename()) + except (NoSectionError, NoOptionError), e: + logger.info('using default logger') + logger.warning(e) + return logging.getLogger('g-pypi') + +def setup_config(cls=None): + """Read config file, create if it doesn't exist""" + if not cls: + cls = Parser + config = cls() + config.add_section('options') + config.set('options', 'background', 'dark') + config.add_section('core') + config.set('core', 'keyword', get_keyword()) + config.set('core', 'overlay', get_portdir_overlay()) + config.set('core', 'overlay_path', '') + config.set('core', 'format', 'ansi') + config.read([get_rc_filename()]) + return config + +class Parser (_Parser): + "Extended ConfigParser with `setboolean()` and similar methods." + def setboolean(self, section, option, value): + self.set(section, option, str(value).lower()) + + def get(self, section, option, default=None): + logger = logging.getLogger('g-pypi') + try: + value = _Parser.get(self, section, option) + except (NoSectionError, NoOptionError), e: + logger.warning(str(e)) + value = default + return value + +CONFIG = setup_config() +setup_logger() diff --git a/g_pypi/ebuild.py b/g_pypi/ebuild.py index cde41ee..1dc55a3 100755 --- a/g_pypi/ebuild.py +++ b/g_pypi/ebuild.py @@ -20,8 +20,8 @@ Creates an ebuild """ +from ConfigParser import NoOptionError import re -import sys import os import logging from time import localtime @@ -36,7 +36,7 @@ from pygments.formatters import BBCodeFormatter from g_pypi.portage_utils import (make_overlay_dir, find_s_dir, unpack_ebuild, get_portdir, get_workdir, find_egg_info_dir, valid_cpn, get_installed_ver, get_repo_names) -from g_pypi.config import MyConfig +from g_pypi.config import CONFIG from g_pypi import enamer from g_pypi.__init__ import __version__ as VERSION @@ -64,8 +64,7 @@ class Ebuild: def __init__(self, up_pn, up_pv, download_url): """Setup ebuild variables""" self.pypi_pkg_name = up_pn - self.config = MyConfig.config - self.options = MyConfig.options + self.config = CONFIG self.logger = logging.getLogger("g-pypi") self.metadata = None self.unpacked_dir = None @@ -87,64 +86,59 @@ class Ebuild: 'use': [], 'slot': '0', 's': '', - 'keywords': self.config['keyword'], + 'keywords': self.config.get('core', 'keyword'), 'inherit': ['distutils'], 'esvn_repo_uri': '', } keyword = os.getenv('ACCEPT_KEYWORDS') if keyword: self.vars['keywords'] = keyword - if self.options.subversion: - #Live svn version ebuild - self.options.pv = "9999" + if self.config.getboolean('options', 'live'): + self.logger.info('generate a live ebuild') + # Live version-control ebuild + self.config.set('options', 'pv', '9999') + # TODO: detect VCS self.vars['esvn_repo_uri'] = download_url self.add_inherit("subversion") - ebuild_vars = enamer.get_vars(download_url, up_pn, up_pv, self.options.pn, - self.options.pv, self.options.my_pn, self.options.my_pv) + ebuild_vars = enamer.get_vars( + download_url, up_pn, up_pv, + self.config.get('options', 'pn'), + self.config.get('options', 'pv'), + self.config.get('options', 'my_pn'), + self.config.get('options', 'my_pv')) for key in ebuild_vars.keys(): if not self.vars.has_key(key): self.vars[key] = ebuild_vars[key] self.vars['p'] = '%s-%s' % (self.vars['pn'], self.vars['pv']) + def _get_option(self, name, default=None): + try: + value = self.config.get('options', name) + except NoOptionError: + value = default + return value + def set_metadata(self, metadata): """Set metadata""" if metadata: self.metadata = metadata else: - self.logger.error("Package has no metadata.") - sys.exit(2) + raise ValueError('package has no metadata') def get_ebuild_vars(self, download_url): """Determine variables from SRC_URI""" - if self.options.pn or self.options.pv: - ebuild_vars = enamer.get_vars(download_url, self.vars['pn'], - self.vars['pv'], self.options.pn, self.options.pv) - else: - ebuild_vars = enamer.get_vars(download_url, self.vars['pn'], - self.vars['pv']) - if self.options.my_p: - ebuild_vars['my_p'] = self.options.my_p - - if self.options.my_pv: - ebuild_vars['my_pv'] = self.options.my_pv - - if self.options.my_pn: - ebuild_vars['my_pn'] = self.options.my_pn - - if ebuild_vars.has_key('my_p'): - self.vars['my_p'] = ebuild_vars['my_p'] - self.vars['my_p_raw'] = ebuild_vars['my_p_raw'] - else: - self.vars['my_p'] = '' - self.vars['my_p_raw'] = ebuild_vars['my_p_raw'] - if ebuild_vars.has_key('my_pn'): - self.vars['my_pn'] = ebuild_vars['my_pn'] - else: - self.vars['my_pn'] = '' - if ebuild_vars.has_key('my_pv'): - self.vars['my_pv'] = ebuild_vars['my_pv'] - else: - self.vars['my_pv'] = '' + pn = self._get_option('pn') + pv = self._get_option('pv') + ebuild_vars = enamer.get_vars( + download_url, self.vars['pn'], self.vars['pv'], pn, pv) + ebuild_vars['my_p'] = self._get_option('my_p', '') + ebuild_vars['my_pv'] = self._get_option('my_pv', '') + ebuild_vars['my_pn'] = self._get_option('my_pn', '') + + self.vars['my_p'] = ebuild_vars['my_p'] + self.vars['my_p_raw'] = ebuild_vars['my_p_raw'] + self.vars['my_pn'] = ebuild_vars['my_pn'] + self.vars['my_pv'] = ebuild_vars['my_pv'] self.vars['src_uri'] = ebuild_vars['src_uri'] @@ -428,7 +422,8 @@ class Ebuild: 'src_install': "", 'src_test': "" } - if not self.options.pretend and self.unpacked_dir: # and \ + pretend = self.config.getboolean('options', 'pretend') + if not pretend and self.unpacked_dir: # and \ # not self.options.subversion: self.post_unpack() functions['src_test'] = self.get_src_test() @@ -462,27 +457,23 @@ class Ebuild: """Print ebuild to stdout""" #No command-line set, config file says no formatting self.logger.info("%s/%s-%s" % \ - (self.options.category, self.vars['pn'], + (self._get_option('category'), self.vars['pn'], self.vars['pv'])) - if self.options.format == "none" or \ - (self.config['format'] == "none" and not self.options.format): + fmt = self.config.get('core', 'format') + if fmt == 'none': self.logger.info(self.ebuild_text) return - - background = self.config['background'] - if self.options.format == "html": + elif fmt == 'html': formatter = HtmlFormatter(full=True) - elif self.config['format'] == "bbcode" or \ - self.options.format == "bbcode": + elif fmt == 'bbcode': formatter = BBCodeFormatter() - elif self.options.format == "ansi" or self.config['format'] == "ansi": + elif fmt == 'ansi': + background = self.config.get('options', 'background') formatter = TerminalFormatter(bg=background) else: - #Invalid formatter specified self.logger.info(self.ebuild_text) - print "ERROR - No formatter" - print self.config['format'], self.options.format - return + self.logger.error('invalid formatter: %s' % fmt) + raise ValueError(fmt) self.logger.info(highlight(self.ebuild_text, BashLexer(), formatter, @@ -492,7 +483,8 @@ class Ebuild: def create_ebuild(self): """Write ebuild and update it after unpacking and examining ${S}""" #Need to write the ebuild first so we can unpack it and check for $S - if self.write_ebuild(overwrite=self.options.overwrite): + overwrite = self.config.getboolean('options', 'overwrite') + if self.write_ebuild(overwrite=overwrite): unpack_ebuild(self.ebuild_path) self.update_with_s() #Write ebuild again after unpacking and adding ${S} @@ -510,38 +502,38 @@ class Ebuild: #overwrite be used? return self.requires + def _overlay_path(self): + overlay_path = self.config.get('core', 'overlay_path') + self.logger.debug('overlay_path: %s' % overlay_path) + if not overlay_path: + overlay_name = self.config.get('core', 'overlay') + self.logger.debug('overlay_name: %s' % overlay_name) + overlays = get_repo_names() + try: + overlay_path = overlays[overlay_name] + except KeyError: + self.logger.error( + 'unknown overylay/repository: %s' % overlay_name) + self.logger.info( + 'known overlays: %s' % ', '.join(sorted(overlays.keys()))) + raise + overlay_path = os.path.expanduser(overlay_path) + self.logger.debug('overlay path: %s' % overlay_path) + return overlay_path + def write_ebuild(self, overwrite=False): """Write ebuild file""" #Use command-line overlay if specified, else the one in .g-pyprc - if self.options.overlay: - overlay_name = self.options.overlay - overlays = get_repo_names() - if overlays.has_key(overlay_name): - overlay_path = overlays[overlay_name] - else: - self.logger.error("Couldn't find overylay/repository by that"+ - " name. I know about these:") - for repo in sorted(overlays.keys()): - self.logger.error(" " + repo.ljust(18) + overlays[repo]) - sys.exit(1) - else: - overlay_path = self.config['overlay'] - ebuild_dir = make_overlay_dir(self.options.category, self.vars['pn'], \ - overlay_path) - if not ebuild_dir: - self.logger.error("Couldn't create overylay ebuild directory.") - sys.exit(2) + overlay_path = self._overlay_path() + ebuild_dir = make_overlay_dir( + self._get_option('category'), self.vars['pn'], overlay_path) self.ebuild_path = os.path.join(ebuild_dir, "%s.ebuild" % \ self.vars['p']) if os.path.exists(self.ebuild_path) and not overwrite: #self.logger.error("Ebuild exists. Use -o to overwrite.") self.logger.warn("Ebuild exists, skipping: %s" % self.ebuild_path) return - try: - out = open(self.ebuild_path, "w") - except IOError, err: - self.logger.error(err) - sys.exit(2) + out = open(self.ebuild_path, "w") out.write(self.ebuild_text) out.close() return True @@ -556,13 +548,13 @@ class Ebuild: #if self.options.subversion: # return self.logger.debug("Trying to determine ${S}, unpacking...") - unpacked_dir = find_s_dir(self.vars['p'], self.options.category) + unpacked_dir = find_s_dir(self.vars['p'], self._get_option('category')) if unpacked_dir == "": self.vars["s"] = "${WORKDIR}" return self.unpacked_dir = os.path.join(get_workdir(self.vars['p'], - self.options.category), unpacked_dir) + self._get_option('category')), unpacked_dir) if unpacked_dir and unpacked_dir != self.vars['p']: if unpacked_dir == self.vars['my_p_raw']: unpacked_dir = '${MY_P}' diff --git a/g_pypi/portage_utils.py b/g_pypi/portage_utils.py index 1a27bd6..77001e7 100644 --- a/g_pypi/portage_utils.py +++ b/g_pypi/portage_utils.py @@ -22,9 +22,13 @@ from portage import settings as portage_settings try: #portage >= 2.2 from portage import dep as portage_dep + from portage import doebuild # perhaps this has moved? + from portage import config as portage_config # perhaps this has moved? except ImportError: #portage <= 2.1 from portage import portage_dep + from portage import doebuild + from portage import config as portage_config try: import gentoolkit # gentoolkit since 0.3.0 are installed as usual @@ -43,9 +47,10 @@ except ImportError: # earlier versions __docformat__ = 'restructuredtext' ENV = portage_config(clone=portage_settings) -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.DEBUG) -LOGGER.addHandler(logging.StreamHandler()) +LOGGER = logging.getLogger('g-pypi') +#_L = logging.getLogger('portage_utils) +#_L.setLevel(LOGGER.logging.DEBUG) +#LOGGER.addHandler(logging.StreamHandler()) def get_repo_names(): @@ -140,13 +145,16 @@ def unpack_ebuild(ebuild_path): """ Use portage to unpack an ebuild - Some day I'll figure out how to get portage to do this directly. Some day. - @param ebuild_path: full path to ebuild @type ebuild_path: string @returns: None if succeed, raises OSError if fails to unpack """ + #for phase in ['digest', 'setup', 'clean', 'unpack']: + # a = portage.doebuild( + # ebuild=ebuild_path, mydo=phase, + # mysettings=-!-!-! portage_config) + (status, output) = commands.getstatusoutput("ebuild %s digest setup clean unpack" % ebuild_path) if status: #Portage's error message, sometimes. @@ -255,12 +263,7 @@ def make_overlay_dir(category, pn, overlay): ebuild_dir = os.path.join(overlay, category, pn) if not os.path.isdir(ebuild_dir): - try: - os.makedirs(ebuild_dir) - except OSError, err: - #XXX Use logger - LOGGER.error(err) - sys.exit(2) + os.makedirs(ebuild_dir) return ebuild_dir diff --git a/scripts/g-pypi b/scripts/g-pypi index 89f6bde..6d21a83 100755 --- a/scripts/g-pypi +++ b/scripts/g-pypi @@ -1,12 +1,17 @@ #!/usr/bin/env python +import logging +import sys + from g_pypi.cli import main +logger = logging.getLogger('g-pypi') + try: main() except Exception, e: - print e + logger.error(e) raise # for debugging sys.exit(1) sys.exit(0) -- 2.26.2