2 # pylint: disable-msg=C0103,C0301,E0611,W0511
4 # Reasons for pylint disable-msg's
6 # E0611 - No name 'resource_string' in module 'pkg_resources'
7 # No name 'BashLexer' in module 'pygments.lexers'
8 # No name 'TerminalFormatter' in module 'pygments.formatters'
9 # (False positives ^^^)
10 # C0103 - Variable names too short (p, pn, pv etc.)
11 # (These can be ignored individually with some in-line pylint-foo.)
12 # C0301 - Line too long in some docstrings
23 from ConfigParser import NoOptionError
27 from time import localtime
29 from Cheetah.Template import Template
30 from pkg_resources import resource_string, WorkingSet, Environment, Requirement
31 from pygments import highlight
32 from pygments.lexers import BashLexer
33 from pygments.formatters import TerminalFormatter, HtmlFormatter
34 from pygments.formatters import BBCodeFormatter
36 from g_pypi.portage_utils import (make_overlay_dir, find_s_dir, unpack_ebuild,
37 get_portdir, get_workdir, find_egg_info_dir, valid_cpn,
38 get_installed_ver, get_repo_names)
39 from g_pypi.config import CONFIG
40 from g_pypi import enamer
41 from g_pypi.__init__ import __version__ as VERSION
44 __docformat__ = 'restructuredtext'
45 __revision__ = '$Revision: 214 $'[11:-1].strip()
48 EBUILD_TEMPLATE = 'ebuild.tmpl'
53 Get g-pypi's version and revision
57 return "%s (rev. %s)" % (VERSION, __revision__)
64 def __init__(self, up_pn, up_pv, download_url):
65 """Setup ebuild variables"""
66 self.pypi_pkg_name = up_pn
68 self.logger = logging.getLogger("g-pypi")
70 self.unpacked_dir = None
78 #Variables that will be passed to the Cheetah template
81 'restrict_python_abis': '',
82 'supported_python_versions': '',
91 'keywords': self.config.get('core', 'keyword'),
92 'inherit': ['distutils'],
95 keyword = os.getenv('ACCEPT_KEYWORDS')
97 self.vars['keywords'] = keyword
98 if self.config.getboolean('options', 'live'):
99 self.logger.info('generate a live ebuild')
100 # Live version-control ebuild
101 self.config.set('options', 'pv', '9999')
103 self.vars['esvn_repo_uri'] = download_url
104 self.add_inherit("subversion")
105 ebuild_vars = enamer.get_vars(
106 download_url, up_pn, up_pv,
107 self.config.get('options', 'pn'),
108 self.config.get('options', 'pv'),
109 self.config.get('options', 'my_pn'),
110 self.config.get('options', 'my_pv'))
111 for key in ebuild_vars.keys():
112 if not self.vars.has_key(key):
113 self.vars[key] = ebuild_vars[key]
114 self.vars['p'] = '%s-%s' % (self.vars['pn'], self.vars['pv'])
116 def _get_option(self, name, default=None):
118 value = self.config.get('options', name)
119 except NoOptionError:
123 def set_metadata(self, metadata):
126 self.metadata = metadata
128 raise ValueError('package has no metadata')
130 def get_ebuild_vars(self, download_url):
131 """Determine variables from SRC_URI"""
132 pn = self._get_option('pn')
133 pv = self._get_option('pv')
134 ebuild_vars = enamer.get_vars(
135 download_url, self.vars['pn'], self.vars['pv'], pn, pv)
136 ebuild_vars['my_p'] = self._get_option('my_p', '')
137 ebuild_vars['my_pv'] = self._get_option('my_pv', '')
138 ebuild_vars['my_pn'] = self._get_option('my_pn', '')
140 self.vars['my_p'] = ebuild_vars['my_p']
141 self.vars['my_p_raw'] = ebuild_vars['my_p_raw']
142 self.vars['my_pn'] = ebuild_vars['my_pn']
143 self.vars['my_pv'] = ebuild_vars['my_pv']
144 self.vars['src_uri'] = ebuild_vars['src_uri']
146 def _quote(self, string):
147 """Escape double quotes to ensure valid ebuild syntax.
149 return string.replace('"', '%22')
151 def add_metadata(self):
153 Extract DESCRIPTION, HOMEPAGE, LICENSE ebuild variables from metadata
155 #Various spellings for 'homepage'
156 homepages = ['Home-page', 'home_page', 'home-page']
157 for hpage in homepages:
158 if self.metadata.has_key(hpage):
159 self.vars['homepage'] = self._quote(self.metadata[hpage])
161 #There doesn't seem to be any specification for case
162 if self.metadata.has_key('Summary'):
163 self.vars['description'] = self.metadata['Summary']
164 elif self.metadata.has_key('summary'):
165 self.vars['description'] = self.metadata['summary']
166 #Replace double quotes to keep bash syntax correct
167 if self.vars['description'] is None:
168 self.vars['description'] = ""
170 self.vars['description'] = self.vars['description'].replace('"', "'")
173 if self.metadata.has_key('classifiers'):
174 for data in self.metadata['classifiers']:
175 if data.startswith("License :: "):
176 my_license = get_portage_license(data)
178 if self.metadata.has_key('License'):
179 my_license = self.metadata['License']
180 elif self.metadata.has_key('license'):
181 my_license = self.metadata['license']
182 my_license = "%s" % my_license
183 if not is_valid_license(my_license):
184 if "LGPL" in my_license:
185 my_license = "LGPL-2.1"
186 elif "GPL" in my_license:
189 self.add_warning("Invalid LICENSE.")
191 self.vars['license'] = "%s" % my_license
193 self.get_python_depend()
195 def get_python_depend(self):
196 """Generate PYTHON_DEPEND and RESTRICT_PYTHON_ABIS strings
198 possible_versions = [(2, i) for i in range(2, 8)]
199 possible_versions.extend([(3, i) for i in range(3)])
201 allowed_versions = []
202 allowed_version_strings = []
203 start = 'Programming Language :: Python ::'
204 for data in self.metadata.get('classifiers', []):
205 if data.startswith(start):
206 version = data[len(start):].strip()
207 allowed_version_strings.append(version)
208 allowed_versions.append(
209 [int(x) for x in version.split('.')])
211 # TODO: also use PEP 345's Requires-Python
213 self.logger.info('supported Python versions: %s' %
214 ', '.join(sorted(allowed_version_strings)))
215 major_versions = set([v[0] for v in allowed_versions])
216 v3 = 3 in major_versions
217 v2 = 2 in major_versions
219 # HACK! need an algorithm for determining these
220 self.vars['python_depend'] = '*:2.6'
221 self.vars['restrict_python_abis'] = '3.*'
222 self.vars['supported_python_versions'] = ', '.join(
223 sorted(allowed_version_strings))
225 def add_warning(self, warning):
226 """Add warning to be shown after ebuild is created"""
227 if warning not in self.warnings:
228 self.warnings.append(warning.lstrip())
230 def post_unpack(self):
231 """Check setup.py for:
232 * PYTHON_MODNAME != $PN
233 * setuptools install_requires or extra_requires
234 # regex: install_requires[ \t]*=[ \t]*\[.*\],
237 name_regex = re.compile('''.*name\s*=\s*[',"]([\w+,\-,\_]*)[',"].*''')
239 re.compile('''.*packages\s*=\s*\[[',"]([\w+,\-,\_]*)[',"].*''')
240 if os.path.exists(self.unpacked_dir):
241 setup_file = os.path.join(self.unpacked_dir, "setup.py")
242 if not os.path.exists(setup_file):
243 self.add_warning("No setup.py found!")
246 self.setup = open(setup_file, "r").readlines()
248 setuptools_requires = module_name = package_name = None
249 for line in self.setup:
250 name_match = name_regex.match(line)
252 package_name = name_match.group(1)
253 elif "packages=" in line or "packages =" in line:
254 #XXX If we have more than one and only one is a top-level
255 #use it e.g. "module, not module.foo, module.bar"
256 mods = line.split(",")[0]
258 # self.add_warning(line)
259 module_match = module_regex.match(mods)
261 module_name = module_match.group(1)
262 elif ("setuptools" in line) and ("import" in line):
263 setuptools_requires = True
264 #It requires setuptools to install pkg
265 self.add_depend("dev-python/setuptools")
267 if setuptools_requires:
268 self.get_dependencies(setup_file)
270 self.logger.warn("This package does not use setuptools so you will have to determine any dependencies if needed.")
272 if module_name and package_name:
273 # if module_name != package_name:
274 self.vars['python_modname'] = module_name
276 def get_unpacked_dist(self, setup_file):
278 Return pkg_resources Distribution object from unpacked package
280 os.chdir(self.unpacked_dir)
281 os.system("/usr/bin/python %s egg_info" % setup_file)
282 ws = WorkingSet([find_egg_info_dir(self.unpacked_dir)])
284 return env.best_match(Requirement.parse(self.pypi_pkg_name), ws)
286 def get_dependencies(self, setup_file):
288 Generate DEPEND/RDEPEND strings
290 * Run setup.py egg_info so we can get the setuptools requirements
293 * Add the unpacked directory to the WorkingEnvironment
295 * Get a Distribution object for package we are isntalling
297 * Get Requirement object containing dependencies
299 a) Determine if any of the requirements are installed
301 b) If requirements aren't installed, see if we have a matching ebuild
302 with adequate version available
304 * Build DEPEND string based on either a) or b)
308 #`dist` is a pkg_resources Distribution object
309 dist = self.get_unpacked_dist(setup_file)
311 #Should only happen if ebuild had 'install_requires' in it but
312 #for some reason couldn't extract egg_info
313 self.logger.warn("Couldn't acquire Distribution obj for %s" % \
317 for req in dist.requires():
319 pkg_name = req.project_name.lower()
320 if not len(req.specs):
321 self.add_setuptools_depend(req)
322 self.add_rdepend("dev-python/%s" % pkg_name)
324 #No version of requirement was specified so we only add
327 comparator, ver = req.specs[0]
328 self.add_setuptools_depend(req)
329 if len(req.specs) > 1:
330 comparator1, ver = req.specs[0]
331 comparator2, ver = req.specs[1]
332 if comparator1.startswith(">") and \
333 comparator2.startswith("<"):
335 self.add_warning("Couldn't resolve requirements. You will need to make sure the RDEPEND for %s is correct." % req)
337 #Some packages have more than one comparator, i.e. cherrypy
338 #for turbogears has >=2.2,<3.0 which would translate to
339 #portage's =dev-python/cherrypy-2.2*
340 self.logger.warn(" **** Requirement %s has multi-specs ****" % req)
341 self.add_rdepend("dev-python/%s" % pkg_name)
343 #Requirement.specs is a list of (comparator,version) tuples
344 if comparator == "==":
346 if valid_cpn("%sdev-python/%s-%s" % (comparator, pkg_name, ver)):
347 self.add_rdepend("%sdev-python/%s-%s" % (comparator, pkg_name, ver))
350 "Invalid PV in dependency: (Requirement %s) %sdev-python/%s-%s" \
351 % (req, comparator, pkg_name, ver)
353 installed_pv = get_installed_ver("dev-python/%s" % pkg_name)
355 self.add_rdepend(">=dev-python/%s-%s" % \
356 (pkg_name, installed_pv))
358 #If we have it installed, use >= installed version
359 #If package has invalid version and we don't have
360 #an ebuild in portage, just add PN to DEPEND, no
361 #version. This means the dep ebuild will have to
362 #be created by adding --MY_? options using the CLI
363 self.add_rdepend("dev-python/%s" % pkg_name)
366 self.add_warning("Couldn't determine dependency: %s" % req)
368 def add_setuptools_depend(self, req):
370 Add dependency for setuptools requirement
371 After current ebuild is created, we check if portage has an
372 ebuild for the requirement, if not create it.
373 @param req: requirement needed by ebuild
374 @type req: pkg_resources `Requirement` object
376 self.logger.debug("Found dependency: %s " % req)
377 if req not in self.requires:
378 self.requires.append(req)
382 Add src_install for installing docs and examples if found
383 and appropriate USE flags e.g. IUSE='doc examples'
386 doc_dirs = ['doc', 'docs']
387 example_dirs = ['example', 'examples', 'demo', 'demos']
389 have_examples = False
391 for ddir in doc_dirs:
392 if os.path.exists(os.path.join(self.unpacked_dir, ddir)):
397 for edir in example_dirs:
398 if os.path.exists(os.path.join(self.unpacked_dir, edir)):
400 self.add_use("examples")
403 if have_docs or have_examples:
404 src_install += '\tdistutils_src_install\n'
406 src_install += '\tif use doc; then\n'
407 src_install += '\t\tdodoc "${S}"/%s/*\n' % have_docs
408 src_install += '\tfi\n'
410 src_install += '\tif use examples; then\n'
411 src_install += '\t\tinsinto /usr/share/doc/"${PF}"/examples\n'
412 src_install += '\t\tdoins -r "${S}"/%s/*\n' % have_examples
413 src_install += '\tfi'
417 def get_src_test(self):
418 """Create src_test if tests detected"""
419 nose_test = '''\tPYTHONPATH=. "${python}" setup.py nosetests || die "tests failed"'''
420 regular_test = '''\tPYTHONPATH=. "${python}" setup.py test || die "tests failed"'''
422 for line in self.setup:
423 if "nose.collector" in line:
424 self.add_depend("test? ( dev-python/nose )")
426 self.has_tests = True
428 #XXX Search for sub-directories
429 if os.path.exists(os.path.join(self.unpacked_dir,
430 "tests")) or os.path.exists(os.path.join(self.unpacked_dir,
432 self.has_tests = True
435 def add_use(self, use_flag):
437 self.vars['use'].append(use_flag)
439 def add_inherit(self, eclass):
440 """Add inherit eclass"""
441 if eclass not in self.vars['inherit']:
442 self.vars['inherit'].append(eclass)
444 def add_depend(self, depend):
445 """Add DEPEND ebuild variable"""
446 if depend not in self.vars['depend']:
447 self.vars['depend'].append(depend)
449 def add_rdepend(self, rdepend):
450 """Add RDEPEND ebuild variable"""
451 if rdepend not in self.vars['rdepend']:
452 self.vars['rdepend'].append(rdepend)
454 def get_ebuild(self):
455 """Generate ebuild from template"""
463 pretend = self.config.getboolean('options', 'pretend')
464 if not pretend and self.unpacked_dir: # and \
465 # not self.options.subversion:
467 functions['src_test'] = self.get_src_test()
468 functions['src_install'] = self.get_docs()
469 # *_f variables are formatted text ready for ebuild
470 self.vars['depend_f'] = format_depend(self.vars['depend'])
471 self.vars['rdepend_f'] = format_depend(self.vars['rdepend'])
472 self.vars['use_f'] = " ".join(self.vars['use'])
473 self.vars['inherit_f'] = " ".join(self.vars['inherit'])
474 template = resource_string(__name__, EBUILD_TEMPLATE)
476 Template(template, searchList=[self.vars, functions]).respond()
478 def set_variables(self):
480 Ensure all variables needed for ebuild template are set and formatted
483 if self.vars['src_uri'].endswith('.zip') or \
484 self.vars['src_uri'].endswith('.ZIP'):
485 self.add_depend("app-arch/unzip")
486 if self.vars['python_modname'] == self.vars['pn']:
487 self.vars['python_modname'] = ""
488 self.vars['year'] = localtime()[0]
489 #Add homepage, license and description from metadata
491 self.vars['warnings'] = self.warnings
492 self.vars['gpypi_version'] = get_version()
494 def print_ebuild(self):
495 """Print ebuild to stdout"""
496 #No command-line set, config file says no formatting
497 self.logger.info("%s/%s-%s" % \
498 (self._get_option('category'), self.vars['pn'],
500 fmt = self.config.get('core', 'format')
502 self.logger.info(self.ebuild_text)
505 formatter = HtmlFormatter(full=True)
506 elif fmt == 'bbcode':
507 formatter = BBCodeFormatter()
509 background = self.config.get('options', 'background')
510 formatter = TerminalFormatter(bg=background)
512 self.logger.info(self.ebuild_text)
513 self.logger.error('invalid formatter: %s' % fmt)
514 raise ValueError(fmt)
515 self.logger.info(highlight(self.ebuild_text,
521 def create_ebuild(self):
522 """Write ebuild and update it after unpacking and examining ${S}"""
523 #Need to write the ebuild first so we can unpack it and check for $S
524 overwrite = self.config.getboolean('options', 'overwrite')
525 if self.write_ebuild(overwrite=overwrite):
526 unpack_ebuild(self.ebuild_path)
528 #Write ebuild again after unpacking and adding ${S}
530 #Run any tests if found
532 # run_tests(self.ebuild_path)
533 #We must overwrite initial skeleton ebuild
534 self.write_ebuild(overwrite=True)
536 self.logger.info("Your ebuild is here: " + self.ebuild_path)
537 #If ebuild already exists, we don't unpack and get dependencies
538 #because they must exist.
539 #We should add an option to force creating dependencies or should
543 def _overlay_path(self):
544 overlay_path = self.config.get('core', 'overlay_path')
545 self.logger.debug('overlay_path: %s' % overlay_path)
547 overlay_name = self.config.get('core', 'overlay')
548 self.logger.debug('overlay_name: %s' % overlay_name)
549 overlays = get_repo_names()
551 overlay_path = overlays[overlay_name]
554 'unknown overylay/repository: %s' % overlay_name)
556 'known overlays: %s' % ', '.join(sorted(overlays.keys())))
558 overlay_path = os.path.expanduser(overlay_path)
559 self.logger.debug('overlay path: %s' % overlay_path)
562 def write_ebuild(self, overwrite=False):
563 """Write ebuild file"""
564 #Use command-line overlay if specified, else the one in .g-pyprc
565 overlay_path = self._overlay_path()
566 ebuild_dir = make_overlay_dir(
567 self._get_option('category'), self.vars['pn'], overlay_path)
568 self.ebuild_path = os.path.join(ebuild_dir, "%s.ebuild" % \
570 if os.path.exists(self.ebuild_path) and not overwrite:
571 #self.logger.error("Ebuild exists. Use -o to overwrite.")
572 self.logger.warn("Ebuild exists, skipping: %s" % self.ebuild_path)
574 out = open(self.ebuild_path, "w")
575 out.write(self.ebuild_text)
579 def show_warnings(self):
580 """Print warnings for incorrect ebuild syntax"""
581 for warning in self.warnings:
582 self.logger.warn("** Warning: %s" % warning)
584 def update_with_s(self):
585 """Add ${S} to ebuild if needed"""
586 #if self.options.subversion:
588 self.logger.debug("Trying to determine ${S}, unpacking...")
589 unpacked_dir = find_s_dir(self.vars['p'], self._get_option('category'))
590 if unpacked_dir == "":
591 self.vars["s"] = "${WORKDIR}"
594 self.unpacked_dir = os.path.join(get_workdir(self.vars['p'],
595 self._get_option('category')), unpacked_dir)
596 if unpacked_dir and unpacked_dir != self.vars['p']:
597 if unpacked_dir == self.vars['my_p_raw']:
598 unpacked_dir = '${MY_P}'
599 elif unpacked_dir == self.vars['my_pn']:
600 unpacked_dir = '${MY_PN}'
601 elif unpacked_dir == self.vars['pn']:
602 unpacked_dir = '${PN}'
604 self.vars["s"] = "${WORKDIR}/%s" % unpacked_dir
606 def get_portage_license(my_license):
608 Map defined classifier license to Portage license
610 http://cheeseshop.python.org/pypi?%3Aaction=list_classifiers
611 We should probably check this list with every major list, eh?
613 my_license = my_license.split(":: ")[-1:][0]
615 "Aladdin Free Public License (AFPL)": "Aladdin",
616 "Academic Free License (AFL)": "AFL-3.0",
617 "Apache Software License": "Apache-2.0",
618 "Apple Public Source License": "Apple",
619 "Artistic License": "Artistic-2",
620 "BSD License": "BSD-2",
621 "Common Public License": "CPL-1.0",
622 "GNU Free Documentation License (FDL)": "FDL-3",
623 "GNU General Public License (GPL)": "GPL-2",
624 "GNU Library or Lesser General Public License (LGPL)": "LGPL-2.1",
625 "IBM Public License": "IBM",
626 "Intel Open Source License": "Intel",
627 "MIT License": "MIT",
628 "Mozilla Public License 1.0 (MPL)": "MPL",
629 "Mozilla Public License 1.1 (MPL 1.1)": "MPL-1.1",
630 "Nethack General Public License": "nethack",
631 "Open Group Test Suite License": "OGTSL",
632 "Python License (CNRI Python License)": "PYTHON",
633 "Python Software Foundation License": "PSF-2.4",
634 "Qt Public License (QPL)": "QPL",
635 "Sleepycat License": "DB",
636 "Sun Public License": "SPL",
637 "University of Illinois/NCSA Open Source License": "ncsa-1.3",
638 "W3C License": "WC3",
639 "zlib/libpng License": "ZLIB",
640 "Zope Public License": "ZPL",
641 "Public Domain": "public-domain"
643 if known_licenses.has_key(my_license):
644 return known_licenses[my_license]
648 def is_valid_license(my_license):
649 """Check if license string matches a valid one in ${PORTDIR}/licenses"""
650 return os.path.exists(os.path.join(get_portdir(), "licenses", my_license))
653 def format_depend(dep_list):
655 Return a formatted string containing DEPEND/RDEPEND
657 @param dep_list: list of portage-ready dependency strings
658 @return: formatted DEPEND or RDEPEND string ready for ebuild
662 DEPEND="dev-python/foo-1.0
666 * First dep has no tab, has linefeed
667 * Middle deps have tab and linefeed
668 * Last dep has tab, no linefeed
672 if not len(dep_list):
675 output = dep_list[0] + "\n"
676 if len(dep_list) == 1:
677 output = output.rstrip()
678 elif len(dep_list) == 2:
679 output += "\t" + dep_list[-1]
683 for dep in dep_list[1:-1]:
684 middle += "\t%s\n" % dep
685 output += middle + "\t" + dep_list[-1]