From: Fabian Groffen Date: Sat, 27 Sep 2008 16:03:31 +0000 (-0000) Subject: Merged from trunk -r11515:11531 X-Git-Url: http://git.tremily.us/gitweb.cgi?a=commitdiff_plain;h=f2bb07bdb7105278985bc79fa0cd60bb7034d183;p=portage.git Merged from trunk -r11515:11531 | 11516 | Fix typo in from previous commit. | | zmedico | | | 11517 | Implement the new EAPI 2 blocker behavior, including !!atom | | zmedico | sytax which forbids temporary simultaneous installation of | | | conflicting packages. | | 11518 | Document the --debug option. Thanks to Jeremy Olexa | | zmedico | suggesting. | | 11519 | Define __all__ and clean up unused imports found by | | zmedico | pyflakes. | | 11520 | Bug #238157 - Update http_proxy/ftp_proxy docs to show | | zmedico | protocol://, for compat with rubygems. | | 11521 | Document the --ignore-default-opts and --skip-manifest | | zmedico | options. | | 11522 | Implement SRC_URI arrows for EAPI 2. The | | zmedico | portdbapi.getfetchlist() method is now deprecated and there | | | is a new getFetchMap() method that returns a dict which maps | | | each file name to a set of alternative URIs. The | | | portage.fetch() function uses introspection to detect when | | | such a dict is passed in and handles it appropriately, while | | | maintaining backward compatibility if a list of uris is | | | passed in. | | 11523 | Add a "EAPI 2_pre3" section to show what's changed. | | zmedico | | | 11524 | Add test cases for SRC_URI validation. | | zmedico | | | 11525 | Fix some broken logic inside portdbapi.getFetchMap(). | | zmedico | | | 11526 | More SRC_URI validation. | | zmedico | | | 11527 | As requested by jmbsvicetto, add support to StaticFileSet | | zmedico | for sets within subdirectories. This makes it possible to | | | create a set name foo/bar by placing a file named bar inside | | | a directory named foo. By allowing sets to be organized in | | | subdirectories, it will allow us to avoid overcrowding as | | | the number of sets grows. | | 11528 | Fix erroneous variable references in the multiset | | zmedico | 'Redefinition of set' message. | | 11529 | Bug #238251 - Use 'read' instead of $IFS for splitting | | zmedico | newlines in e* functions, since using $IFS causes spurious | | | newlines to be inserted in the message when the e* function | | | is called with more than one argument. | | 11530 | Always call `cd "$WORKDIR"` to ensure that $CWD is sane | | zmedico | whenever possible. | | 11531 | Add a new --list-sets action, as requested by jmbsvicetto. | | zmedico | | svn path=/main/branches/prefix/; revision=11570 --- diff --git a/bin/ebuild.sh b/bin/ebuild.sh index 93c675865..c40e9fc5d 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -272,6 +272,9 @@ register_die_hook() { export EBUILD_DEATH_HOOKS="${EBUILD_DEATH_HOOKS} $*" } +# Ensure that $CWD is sane whenever possible. +cd "$WORKDIR" 2>/dev/null + #if no perms are specified, dirs/files will have decent defaults #(not secretive, but not stupid) umask 022 diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 87ac8ac2f..317e4360f 100755 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -165,7 +165,7 @@ vecho() { # Internal logging function, don't use this in ebuilds elog_base() { - local line lines=0 messagetype + local messagetype [ -z "${1}" -o -z "${T}" -o ! -d "${T}/logging" ] && return 1 case "${1}" in INFO|WARN|ERROR|LOG|QA) @@ -181,39 +181,19 @@ elog_base() { # not entirely safe to use it as a delimiter in the log file since # there can still be escaped newlines that will be expanded due to # the echo -e parameter. - save_IFS - IFS=$'\n' - for line in $* ; do - (( lines++ )) + echo "$@" | while read line ; do echo -ne "${messagetype} ${line}\n\0" >> \ "${T}/logging/${EBUILD_PHASE:-other}" done - restore_IFS - - # This is needed in case a blank line is being shown. - [ $lines -eq 0 ] && \ - echo -ne "${messagetype} $*\n\0" >> \ - "${T}/logging/${EBUILD_PHASE:-other}" - return 0 } eqawarn() { elog_base QA "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - local line lines=0 - save_IFS - IFS=$'\n' - for line in $* ; do - (( lines++ )) + echo "$@" | while read line ; do vecho -e " ${WARN}*${NORMAL} ${line}" >&2 done - restore_IFS - - # This is needed in case a blank line is being shown. - [ $lines -eq 0 ] && \ - vecho -e " ${WARN}*${NORMAL} $*" >&2 - LAST_E_CMD="eqawarn" return 0 } @@ -221,19 +201,9 @@ eqawarn() { elog() { elog_base LOG "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - local line lines=0 - save_IFS - IFS=$'\n' - for line in $* ; do - (( lines++ )) + echo "$@" | while read line ; do echo -e " ${GOOD}*${NORMAL} ${line}" done - restore_IFS - - # This is needed in case a blank line is being shown. - [ $lines -eq 0 ] && \ - echo -e " ${GOOD}*${NORMAL} $*" - LAST_E_CMD="elog" return 0 } @@ -261,19 +231,9 @@ esyslog() { einfo() { elog_base INFO "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - save_IFS - IFS=$'\n' - local line lines=0 - for line in $* ; do - (( lines++ )) + echo "$@" | while read line ; do echo -e " ${GOOD}*${NORMAL} ${line}" done - restore_IFS - - # This is needed in case a blank line is being shown. - [ $lines -eq 0 ] && \ - echo -e " ${GOOD}*${NORMAL} $*" - LAST_E_CMD="einfo" return 0 } @@ -289,19 +249,9 @@ einfon() { ewarn() { elog_base WARN "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - save_IFS - IFS=$'\n' - local line lines=0 - for line in $* ; do - (( lines++ )) + echo "$@" | while read line ; do echo -e " ${WARN}*${NORMAL} ${RC_INDENTATION}${line}" >&2 done - restore_IFS - - # This is needed in case a blank line is being shown. - [ $lines -eq 0 ] && \ - echo -e " ${WARN}*${NORMAL} ${RC_INDENTATION}$*" >&2 - LAST_E_CMD="ewarn" return 0 } @@ -309,19 +259,9 @@ ewarn() { eerror() { elog_base ERROR "$*" [[ ${RC_ENDCOL} != "yes" && ${LAST_E_CMD} == "ebegin" ]] && echo - save_IFS - IFS=$'\n' - local line lines=0 - for line in $* ; do - (( lines++ )) + echo "$@" | while read line ; do echo -e " ${BAD}*${NORMAL} ${RC_INDENTATION}${line}" >&2 done - restore_IFS - - # This is needed in case a blank line is being shown. - [ $lines -eq 0 ] && \ - echo -e " ${BAD}*${NORMAL} ${RC_INDENTATION}$*" >&2 - LAST_E_CMD="eerror" return 0 } @@ -561,7 +501,7 @@ save_ebuild_env() { for x in pkg_setup pkg_nofetch src_unpack src_prepare src_configure \ src_compile src_test src_install pkg_preinst pkg_postinst \ pkg_prerm pkg_postrm ; do - unset -f _default_$x _eapi{0,1,2}_$x + unset -f default_$x _eapi{0,1,2}_$x done unset x diff --git a/bin/repoman b/bin/repoman index 7c6cd0161..6c7b80374 100755 --- a/bin/repoman +++ b/bin/repoman @@ -1255,8 +1255,6 @@ for x in scanlist: badsyntax.append("'%s' not a valid atom" % token) else: is_blocker = atom.blocker - if atom.blocker: - atom = portage.dep.Atom(atom[1:]) if mytype == "DEPEND" and \ not is_blocker and \ @@ -1283,6 +1281,13 @@ for x in scanlist: (relative_path + ": %s use dependency" + \ " not supported with EAPI='%s':" + \ " '%s'") % (mytype, eapi, atom)) + if atom.blocker and atom.blocker.overlap.forbid \ + and eapi in ("0", "1"): + stats['EAPI.incompatible'] += 1 + fails['EAPI.incompatible'].append( + (relative_path + ": %s new blocker syntax" + \ + " not supported with EAPI='%s':" + \ + " '%s'") % (mytype, eapi, atom)) type_list.extend([mytype] * (len(badsyntax) - len(type_list))) diff --git a/doc/package/ebuild/eapi/2.docbook b/doc/package/ebuild/eapi/2.docbook index ceec15ccf..ef1038b40 100644 --- a/doc/package/ebuild/eapi/2.docbook +++ b/doc/package/ebuild/eapi/2.docbook @@ -216,6 +216,67 @@ src_compile() { +
+ EAPI 2_pre3 +
+ Metadata +
+ Dependencies +
+ Blocker Atoms +
+ New Meaning for Old Syntax + + Blocker atoms which use the previously existing !atom syntax now have + a slightly different meaning. These blocker atoms indicate that + conflicting packages may be temporarily installed simultaneously. When + temporary simultaneous installation of conflicting packages occurs, the + installation of a newer package may overwrite any colliding files that + belong to an older package which is explicitly blocked. When such file + collisions occur, the colliding files cease to belong to the older + package, and they remain installed after the older package is + eventually uninstalled. The older package is uninstalled only after + any newer blocking packages have been merged on top of it. + +
+
+ New !!atom Syntax + + A new !!atom syntax is now supported, for use in special cases for which + temporary simultaneous installation of conflicting packages should not be + allowed. If a given package happens to be blocked my a mixture of atoms + consisting of both the !atom and !!atom syntaxes, the !!atom syntax takes + precedence over the !atom syntax. + +
+
+
+
+ SRC_URI +
+ Customization of Output File Names + + A new syntax is supported which allows customization of the output + file name for a given URI. In order to customize the output file + name, a given URI should be followed by a "->" operator which, + in turn, should be followed by the desired output file name. As + usual, all tokens, including the operator and output file name, + should be separated by whitespace. + +
+
+
+
+ Phases +
+ New src_prepare Phase Function + + A new src_prepare function is called after the src_unpack + function, with cwd initially set to $S. + +
+
+
EAPI 2 Draft
diff --git a/man/ebuild.1 b/man/ebuild.1 index 1bbb004e4..1384bb157 100644 --- a/man/ebuild.1 +++ b/man/ebuild.1 @@ -171,11 +171,21 @@ Builds a RedHat RPM package from the files in the temporary information is not incorporated into the RPM. .SH OPTIONS .TP -\fB--force\fR +.BR "\-\-debug" +Run bash with the \-x option, causing it to output verbose debugging +information to stdout. +.TP +.BR "\-\-force" When used together with the digest or manifest command, this option forces regeneration of digests for all distfiles associated with the current ebuild. Any distfiles that do not already exist in ${DISTDIR} will be automatically fetched. +.TP +.BR "\-\-ignore\-default\-opts" +Do not use the \fIEBUILD_DEFAULT_OPTS\fR environment variable. +.TP +.BR "\-\-skip\-manifest" +Skip all manifest checks. .SH "REPORTING BUGS" Please report bugs via http://bugs.gentoo.org/ .SH "AUTHORS" diff --git a/man/emerge.1 b/man/emerge.1 index e4e0533b4..a14cc615c 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -67,7 +67,8 @@ on the current configuration. The default set configuration is located in \fB/usr/share/portage/config/sets.conf\fR. Note that a \fIset\fR is generally used in conjunction with \fB\-\-update\fR. When used as arguments to \fBemerge\fR sets have to be prefixed with \fB@\fR to be -recognized. +recognized. Use the \fB\-\-list\-sets\fR action to display a list of +available package sets. .TP .BR atom An \fIatom\fR describes bounds on a package that you wish to install. @@ -142,6 +143,9 @@ developers when fixing the reported problem. \fBPlease include this information when submitting a bug report.\fR Expanded output can be obtained with the \fI\-\-verbose\fR option. .TP +.BR \-\-list\-sets +Displays a list of available package sets. +.TP .BR \-\-metadata Transfers metadata cache from ${PORTDIR}/metadata/cache/ to /var/cache/edb/dep/ as is normally done on the diff --git a/man/make.conf.5 b/man/make.conf.5 index 054d83d1c..45ad93b0c 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -375,7 +375,7 @@ the \fIebuild scripts\fR. Merging 'mirrorselect' can help. Entries in this variable that have no protocol and simply start with a '/' path separator may be used to specify mounted filesystem mirrors. .TP -\fBhttp_proxy ftp_proxy\fR = \fI[host:port]\fR +\fBhttp_proxy ftp_proxy\fR = \fI[protocol://host:port]\fR These vars are used if the sources must be downloaded from the internet by \fBwget\fR(1). They are only required if you use a proxy server for internet access. diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index bde4a6ca3..f9cbf7202 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -212,12 +212,12 @@ def userquery(prompt, responses=None, colours=None): print "Interrupted." sys.exit(1) -actions=[ +actions = frozenset([ "clean", "config", "depclean", -"info", "metadata", +"info", "list-sets", "metadata", "prune", "regen", "search", "sync", "unmerge", -] +]) options=[ "--ask", "--alphabetical", "--buildpkg", "--buildpkgonly", @@ -446,7 +446,7 @@ class search(object): pass self.portdb = fake_portdb for attrib in ("aux_get", "cp_all", - "xmatch", "findname", "getfetchlist"): + "xmatch", "findname", "getFetchMap"): setattr(fake_portdb, attrib, getattr(self, "_"+attrib)) self._dbs = [] @@ -493,14 +493,14 @@ class search(object): return value return None - def _getfetchlist(self, *args, **kwargs): + def _getFetchMap(self, *args, **kwargs): for db in self._dbs: - func = getattr(db, "getfetchlist", None) + func = getattr(db, "getFetchMap", None) if func: value = func(*args, **kwargs) if value: return value - return [], [] + return {} def _visible(self, db, cpv, metadata): installed = db is self.vartree.dbapi @@ -699,8 +699,7 @@ class search(object): from portage import manifest mf = manifest.Manifest( pkgdir, self.settings["DISTDIR"]) - fetchlist = self.portdb.getfetchlist(mycpv, - mysettings=self.settings, all=True)[1] + fetchlist = self.portdb.getFetchMap(mycpv) try: mysum[0] = mf.getDistfilesSize(fetchlist) except KeyError, e: @@ -1413,7 +1412,7 @@ class Task(SlotObject): class Blocker(Task): __hash__ = Task.__hash__ - __slots__ = ("root", "atom", "cp", "satisfied") + __slots__ = ("root", "atom", "cp", "eapi", "satisfied") def __init__(self, **kwargs): Task.__init__(self, **kwargs) @@ -1423,7 +1422,7 @@ class Blocker(Task): hash_key = getattr(self, "_hash_key", None) if hash_key is None: self._hash_key = \ - ("blocks", self.root, self.atom) + ("blocks", self.root, self.atom, self.eapi) return self._hash_key class Package(Task): @@ -4411,7 +4410,9 @@ class depgraph(object): return 1 # The blocker applies to the root where # the parent is or will be installed. - blocker = Blocker(atom=dep.atom, root=dep.parent.root) + blocker = Blocker(atom=dep.atom, + eapi=dep.parent.metadata["EAPI"], + root=dep.parent.root) self._blocker_parents.add(blocker, dep.parent) return 1 dep_pkg, existing_node = self._select_package(dep.root, dep.atom, @@ -4669,15 +4670,14 @@ class depgraph(object): for atom in selected_atoms: try: - blocker = atom.startswith("!") - if blocker: - atom = atom[1:] + atom = portage.dep.Atom(atom) + mypriority = dep_priority.copy() - if not blocker and vardb.match(atom): + if not atom.blocker and vardb.match(atom): mypriority.satisfied = True if not self._add_dep(Dependency(atom=atom, - blocker=blocker, depth=depth, parent=pkg, + blocker=atom.blocker, depth=depth, parent=pkg, priority=mypriority, root=dep_root), allow_unsatisfied=allow_unsatisfied): return 0 @@ -5833,7 +5833,7 @@ class depgraph(object): except KeyError: pass if blockers is not None: - blockers = set("!" + blocker.atom \ + blockers = set(str(blocker.atom) \ for blocker in blockers) # If this node has any blockers, create a "nomerge" @@ -5911,7 +5911,8 @@ class depgraph(object): blocker_cache.BlockerData(counter, blocker_atoms) if blocker_atoms: for myatom in blocker_atoms: - blocker = Blocker(atom=myatom[1:], root=myroot) + blocker = Blocker(atom=portage.dep.Atom(myatom), + eapi=pkg.metadata["EAPI"], root=myroot) self._blocker_parents.add(blocker, pkg) for cpv in stale_cache: del blocker_cache[cpv] @@ -5930,7 +5931,7 @@ class depgraph(object): self.spinner.update() root_config = self.roots[blocker.root] virtuals = root_config.settings.getvirtuals() - mytype, myroot, mydep = blocker + myroot = blocker.root initial_db = self.trees[myroot]["vartree"].dbapi final_db = self.mydbapi[myroot] @@ -6401,7 +6402,18 @@ class depgraph(object): if self.digraph.contains(inst_pkg): continue - if running_root == task.root: + forbid_overlap = False + heuristic_overlap = False + for blocker in myblocker_uninstalls.parent_nodes(task): + if blocker.eapi in ("0", "1"): + heuristic_overlap = True + elif blocker.atom.blocker.overlap.forbid: + forbid_overlap = True + break + if forbid_overlap and running_root == task.root: + continue + + if heuristic_overlap and running_root == task.root: # Never uninstall sys-apps/portage or it's essential # dependencies, except through replacement. try: @@ -6601,7 +6613,8 @@ class depgraph(object): # will be temporarily installed simultaneously. for blocker in solved_blockers: retlist.append(Blocker(atom=blocker.atom, - root=blocker.root, satisfied=True)) + root=blocker.root, eapi=blocker.eapi, + satisfied=True)) unsolvable_blockers = set(self._unsolvable_blockers.leaf_nodes()) for node in myblocker_uninstalls.root_nodes(): @@ -6970,7 +6983,7 @@ class depgraph(object): if x.satisfied: counters.blocks_satisfied += 1 resolved = portage.key_expand( - pkg_key, mydb=vardb, settings=pkgsettings) + str(x.atom).lstrip("!"), mydb=vardb, settings=pkgsettings) if "--columns" in self.myopts and "--quiet" in self.myopts: addl += " " + colorize(blocker_style, resolved) else: @@ -6983,7 +6996,7 @@ class depgraph(object): if resolved!=x[2]: addl += colorize(blocker_style, " (\"%s\" is blocking %s)") % \ - (pkg_key, block_parents) + (str(x.atom).lstrip("!"), block_parents) else: addl += colorize(blocker_style, " (is blocking %s)") % block_parents @@ -13595,10 +13608,15 @@ def emerge_main(): print colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n") break + root_config = trees[settings["ROOT"]]["root_config"] + if myaction == "list-sets": + sys.stdout.write("".join("%s\n" % s for s in sorted(root_config.sets))) + sys.stdout.flush() + return os.EX_OK + # only expand sets for actions taking package arguments oldargs = myfiles[:] if myaction in ("clean", "config", "depclean", "info", "prune", "unmerge", None): - root_config = trees[settings["ROOT"]]["root_config"] setconfig = root_config.setconfig # display errors that occured while loading the SetConfig instance for e in setconfig.errors: @@ -13840,8 +13858,6 @@ def emerge_main(): "not support '--pretend'.\n") % myaction) return 1 - root_config = trees[settings["ROOT"]]["root_config"] - if "sync" == myaction: return action_sync(settings, trees, mtimedb, myopts, myaction) elif "metadata" == myaction: diff --git a/pym/_emerge/help.py b/pym/_emerge/help.py index e0b93480c..5c7b0d6e6 100644 --- a/pym/_emerge/help.py +++ b/pym/_emerge/help.py @@ -20,7 +20,7 @@ def shorthelp(): print " [ "+green("--jobs") + " " + turquoise("JOBS")+" ] [ "+green("--keep-going")+" ] [ " + green("--load-average")+" " + turquoise("LOAD") + " ]" print " [ "+green("--newuse")+" ] [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ] [ "+green("--oneshot")+" ]" print " [ "+green("--reinstall ")+turquoise("changed-use")+" ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]" - print bold("Actions:")+" [ "+green("--clean")+" | "+green("--depclean")+" | "+green("--prune")+" | "+green("--regen")+" | "+green("--search")+" | "+green("--unmerge")+" ]" + print bold("Actions:")+" [ "+green("--depclean")+" | "+green("--list-sets")+" | "+green("--search")+" | "+green("--sync")+" | "+green("--version")+" ]" def help(myaction,myopts,havecolor=1): # TODO: Implement a wrap() that accounts for console color escape codes. @@ -112,6 +112,12 @@ def help(myaction,myopts,havecolor=1): print " make.{conf,globals,defaults} and the environment show up if" print " run with the '--verbose' flag." print + print " " + green("--list-sets") + paragraph = "Displays a list of available package sets." + + for line in wrap(paragraph, desc_width): + print desc_indent + line + print print " "+green("--metadata") print " Transfers metadata cache from ${PORTDIR}/metadata/cache/ to" print " /var/cache/edb/dep/ as is normally done on the tail end of an" diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index a8f541148..278aacf44 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -3507,11 +3507,19 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locks_in_subdir=".locks", else: locations = mymirrors + file_uri_tuples = [] + if isinstance(myuris, dict): + for myfile, uri_set in myuris.iteritems(): + for myuri in uri_set: + file_uri_tuples.append((myfile, myuri)) + else: + for myuri in myuris: + file_uri_tuples.append((os.path.basename(myuri), myuri)) + filedict={} primaryuri_indexes={} primaryuri_dict = {} - for myuri in myuris: - myfile=os.path.basename(myuri) + for myfile, myuri in file_uri_tuples: if myfile not in filedict: filedict[myfile]=[] for y in range(0,len(locations)): @@ -4170,11 +4178,8 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None doebuild_environment(myebuild, "fetch", mysettings["ROOT"], fetch_settings, debug, 1, myportdb) - alluris, aalist = myportdb.getfetchlist( - cpv, mytree=mytree, all=True, - mysettings=fetch_settings) - myuris = [uri for uri in alluris \ - if os.path.basename(uri) == myfile] + uri_map = myportdb.getFetchMap(cpv, mytree=mytree) + myuris = {myfile:uri_map[myfile]} fetch_settings["A"] = myfile # for use by pkg_nofetch() if fetch(myuris, fetch_settings): success = True @@ -4211,8 +4216,7 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None writemsg_stdout(" digest.assumed" + portage.output.colorize("WARN", str(len(auto_assumed)).rjust(18)) + "\n") for pkg_key in pkgs: - fetchlist = myportdb.getfetchlist(pkg_key, - mysettings=mysettings, all=True, mytree=mytree)[1] + fetchlist = myportdb.getFetchMap(pkg_key, mytree=mytree) pv = pkg_key.split("/")[1] for filename in auto_assumed: if filename in fetchlist: @@ -5628,11 +5632,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Make sure we get the correct tree in case there are overlays. mytree = os.path.realpath( os.path.dirname(os.path.dirname(mysettings["O"]))) + useflags = mysettings["PORTAGE_USE"].split() try: - newuris, alist = mydbapi.getfetchlist( - mycpv, mytree=mytree, mysettings=mysettings) - alluris, aalist = mydbapi.getfetchlist( - mycpv, mytree=mytree, all=True, mysettings=mysettings) + alist = mydbapi.getFetchMap(mycpv, useflags=useflags, mytree=mytree) + aalist = mydbapi.getFetchMap(mycpv, mytree=mytree) except portage.exception.InvalidDependString, e: writemsg("!!! %s\n" % str(e), noiselevel=-1) writemsg("!!! Invalid SRC_URI for '%s'.\n" % mycpv, noiselevel=-1) @@ -5641,26 +5644,11 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, mysettings["A"] = " ".join(alist) mysettings["AA"] = " ".join(aalist) if ("mirror" in features) or fetchall: - fetchme = alluris[:] - checkme = aalist[:] - elif mydo == "digest": - fetchme = alluris[:] - checkme = aalist[:] - # Skip files that we already have digests for. - mf = Manifest(mysettings["O"], mysettings["DISTDIR"]) - mydigests = mf.getTypeDigests("DIST") - required_hash_types = set() - required_hash_types.add("size") - required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) - for filename, hashes in mydigests.iteritems(): - if not required_hash_types.difference(hashes): - checkme = [i for i in checkme if i != filename] - fetchme = [i for i in fetchme \ - if os.path.basename(i) != filename] - del filename, hashes + fetchme = aalist + checkme = aalist else: - fetchme = newuris[:] - checkme = alist[:] + fetchme = alist + checkme = alist if mydo == "fetch": # Files are already checked inside fetch(), @@ -6919,8 +6907,7 @@ class FetchlistDict(UserDict.DictMixin): self.portdb = mydbapi def __getitem__(self, pkg_key): """Returns the complete fetch list for a given package.""" - return self.portdb.getfetchlist(pkg_key, mysettings=self.settings, - all=True, mytree=self.mytree)[1] + return self.portdb.getFetchMap(pkg_key, mytree=self.mytree).keys() def __contains__(self, cpv): return cpv in self.keys() def has_key(self, pkg_key): diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 614f6dec3..33c8b52f2 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -2,29 +2,84 @@ # Distributed under the terms of the GNU General Public License v2 # $Id$ +__all__ = ["portdbapi", "close_portdbapi_caches", "portagetree"] + from portage.cache.cache_errors import CacheError from portage.cache.mappings import slot_dict_class from portage.const import REPO_NAME_LOC from portage.data import portage_gid, secpass from portage.dbapi import dbapi -from portage.dep import use_reduce, paren_reduce, dep_getslot, dep_getkey, \ - match_from_list, match_to_list, remove_slot -from portage.exception import OperationNotPermitted, PortageException, \ +from portage.dep import use_reduce, paren_reduce, dep_getkey, match_from_list +from portage.exception import PortageException, \ UntrustedSignature, SecurityViolation, InvalidSignature, MissingSignature, \ FileNotFound, InvalidDependString, InvalidPackageName from portage.manifest import Manifest -from portage.output import red -from portage.util import ensure_dirs, writemsg, apply_recursive_permissions -from portage.versions import pkgcmp, pkgsplit, catpkgsplit, best, ver_regexp +from portage.util import ensure_dirs, writemsg +from portage.versions import pkgsplit, catpkgsplit, best, ver_regexp import portage.gpg, portage.checksum -from portage import eclass_cache, auxdbkeys, auxdbkeylen, doebuild, flatten, \ +from portage import eclass_cache, auxdbkeys, doebuild, flatten, \ listdir, dep_expand, eapi_is_supported, key_expand, dep_check -import os, re, stat, sys +import os, stat from itertools import izip +def _src_uri_validate(cpv, eapi, src_uri): + """ + Take a SRC_URI structure as returned by paren_reduce or use_reduce + and validate it. Raises InvalidDependString if a problem is detected, + such as missing operand for a -> operator. + """ + uri = None + operator = None + for x in src_uri: + if isinstance(x, list): + if operator is not None: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "right operand") % (cpv,)) + uri = None + _src_uri_validate(cpv, eapi, x) + continue + if x[:-1] == "?": + if operator is not None: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "right operand") % (cpv,)) + uri = None + continue + if uri is None: + if x == "->": + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "left operand") % (cpv,)) + uri = x + continue + if x == "->": + if eapi in ("0", "1"): + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrows are not " + \ + "supported with EAPI='%s'") % (cpv, eapi)) + operator = x + continue + if operator is not None: + if "/" in x: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI '/' character in " + \ + "file name: '%s'") % (cpv, x)) + if x[-1] == "?": + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI arrow missing " + \ + "right operand") % (cpv,)) + uri = None + operator = None + + if operator is not None: + raise portage.exception.InvalidDependString( + "getFetchMap(): '%s' SRC_URI arrow missing right operand" % \ + (cpv,)) + class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] @@ -450,9 +505,24 @@ class portdbapi(dbapi): return returnme - def getfetchlist(self, mypkg, useflags=None, mysettings=None, all=0, mytree=None): - if mysettings is None: - mysettings = self.doebuild_settings + def getFetchMap(self, mypkg, useflags=None, mytree=None): + """ + Get the SRC_URI metadata as a dict which maps each file name to a + set of alternative URIs. + + @param mypkg: cpv for an ebuild + @type mypkg: String + @param useflags: a collection of enabled USE flags, for evaluation of + conditionals + @type useflags: set, or None to enable all conditionals + @param mytree: The canonical path of the tree in which the ebuild + is located, or None for automatic lookup + @type mypkg: String + @returns: A dict which maps each file name to a set of alternative + URIs. + @rtype: dict + """ + try: eapi, myuris = self.aux_get(mypkg, ["EAPI", "SRC_URI"], mytree=mytree) @@ -460,33 +530,64 @@ class portdbapi(dbapi): # Convert this to an InvalidDependString exception since callers # already handle it. raise portage.exception.InvalidDependString( - "getfetchlist(): aux_get() error reading "+mypkg+"; aborting.") + "getFetchMap(): aux_get() error reading "+mypkg+"; aborting.") if not eapi_is_supported(eapi): # Convert this to an InvalidDependString exception # since callers already handle it. raise portage.exception.InvalidDependString( - "getfetchlist(): '%s' has unsupported EAPI: '%s'" % \ + "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ (mypkg, eapi.lstrip("-"))) - if not all and useflags is None: - mysettings.setcpv(mypkg, mydb=self) - useflags = mysettings["PORTAGE_USE"].split() + myuris = paren_reduce(myuris) + _src_uri_validate(mypkg, eapi, myuris) + myuris = use_reduce(myuris, uselist=useflags, + matchall=(useflags is None)) + myuris = flatten(myuris) - myurilist = paren_reduce(myuris) - myurilist = use_reduce(myurilist, uselist=useflags, matchall=all) - newuris = flatten(myurilist) + uri_map = {} - myfiles = [] - for x in newuris: - mya = os.path.basename(x) - if not mya: - raise portage.exception.InvalidDependString( - "getfetchlist(): '%s' SRC_URI has no file name: '%s'" % \ - (mypkg, x)) - if not mya in myfiles: - myfiles.append(mya) - return [newuris, myfiles] + myuris.reverse() + while myuris: + uri = myuris.pop() + if myuris and myuris[-1] == "->": + operator = myuris.pop() + distfile = myuris.pop() + else: + distfile = os.path.basename(uri) + if not distfile: + raise portage.exception.InvalidDependString( + ("getFetchMap(): '%s' SRC_URI has no file " + \ + "name: '%s'") % (mypkg, uri)) + + uri_set = uri_map.get(distfile) + if uri_set is None: + uri_set = set() + uri_map[distfile] = uri_set + uri_set.add(uri) + uri = None + operator = None + + return uri_map + + def getfetchlist(self, mypkg, useflags=None, mysettings=None, + all=0, mytree=None): + + writemsg("!!! pordbapi.getfetchlist() is deprecated, " + \ + "use getFetchMap() instead.\n", noiselevel=-1) + + if all: + useflags = None + elif useflags is None: + if mysettings is None: + mysettings = self.doebuild_settings + mysettings.setcpv(mypkg, mydb=self) + useflags = mysettings["PORTAGE_USE"].split() + uri_map = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree) + uris = set() + for uri_set in uri_map.itervalues(): + uris.update(uri_set) + return [list(uris), uri_map.keys()] def getfetchsizes(self, mypkg, useflags=None, debug=0): # returns a filename:size dictionnary of remaining downloads @@ -499,10 +600,7 @@ class portdbapi(dbapi): print "[empty/missing/bad digest]: "+mypkg return None filesdict={} - if useflags is None: - myuris, myfiles = self.getfetchlist(mypkg,all=1) - else: - myuris, myfiles = self.getfetchlist(mypkg,useflags=useflags) + myfiles = self.getFetchMap(mypkg, useflags=useflags) #XXX: maybe this should be improved: take partial downloads # into account? check checksums? for myfile in myfiles: @@ -530,10 +628,12 @@ class portdbapi(dbapi): return filesdict def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False): - if not useflags: + if all: + useflags = None + elif useflags is None: if mysettings: useflags = mysettings["USE"].split() - myuri, myfiles = self.getfetchlist(mypkg, useflags=useflags, mysettings=mysettings, all=all) + myfiles = self.getFetchMap(mypkg, useflags=useflags) myebuild = self.findname(mypkg) pkgdir = os.path.dirname(myebuild) mf = Manifest(pkgdir, self.mysettings["DISTDIR"]) diff --git a/pym/portage/dep.py b/pym/portage/dep.py index adbde5f6d..b36a6bf5a 100644 --- a/pym/portage/dep.py +++ b/pym/portage/dep.py @@ -489,14 +489,34 @@ class Atom(object): __slots__ = ("__weakref__", "blocker", "cp", "cpv", "operator", "slot", "use") + _str_methods + class _blocker(object): + __slots__ = ("overlap",) + + class _overlap(object): + __slots__ = ("forbid",) + + def __init__(self, forbid=False): + self.forbid = forbid + + def __init__(self, forbid_overlap=False): + self.overlap = self._overlap(forbid=forbid_overlap) + def __init__(self, s): if not isvalidatom(s, allow_blockers=True): raise InvalidAtom(s) for x in self._str_methods: setattr(self, x, getattr(s, x)) - self.blocker = "!" == s[:1] - if self.blocker: - s = s[1:] + + blocker = "!" == s[:1] + if blocker: + self.blocker = self._blocker(forbid_overlap=("!" == s[1:2])) + if self.blocker.overlap.forbid: + s = s[2:] + else: + s = s[1:] + else: + self.blocker = False + self.cp = dep_getkey(s) self.cpv = dep_getcpv(s) self.slot = dep_getslot(s) @@ -586,7 +606,10 @@ def dep_getcpv(mydep): if mydep and mydep[-1] == "*": mydep = mydep[:-1] if mydep and mydep[0] == "!": - mydep = mydep[1:] + if mydep[1:2] == "!": + mydep = mydep[2:] + else: + mydep = mydep[1:] if mydep[:2] in [">=", "<="]: mydep = mydep[2:] elif mydep[:1] in "=<>~": @@ -720,8 +743,11 @@ def isvalidatom(atom, allow_blockers=False): global _invalid_atom_chars_regexp if _invalid_atom_chars_regexp.search(atom): return 0 - if allow_blockers and atom.startswith("!"): - atom = atom[1:] + if allow_blockers and atom[:1] == "!": + if atom[1:2] == "!": + atom = atom[2:] + else: + atom = atom[1:] try: use = dep_getusedeps(atom) diff --git a/pym/portage/sets/__init__.py b/pym/portage/sets/__init__.py index d688164b5..f88e1b6ee 100644 --- a/pym/portage/sets/__init__.py +++ b/pym/portage/sets/__init__.py @@ -65,7 +65,7 @@ class SetConfig(SafeConfigParser): continue for x in newsets: if x in self.psets: - self.errors.append("Redefinition of set '%s' (sections: '%s', '%s')" % (setname, self.psets[setname].creator, sname)) + self.errors.append("Redefinition of set '%s' (sections: '%s', '%s')" % (x, self.psets[x].creator, sname)) newsets[x].creator = sname if self.has_option(sname, "world-candidate") and not self.getboolean(sname, "world-candidate"): newsets[x].world_candidate = False diff --git a/pym/portage/sets/files.py b/pym/portage/sets/files.py index 8bcf131fa..9dfc667a1 100644 --- a/pym/portage/sets/files.py +++ b/pym/portage/sets/files.py @@ -6,7 +6,7 @@ import os import re from itertools import chain -from portage.util import grabfile, write_atomic, ensure_dirs +from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path from portage.const import PRIVATE_PATH, USER_CONFIG_PATH, EPREFIX_LSTRIP from portage.locks import lockfile, unlockfile from portage import portage_gid @@ -122,12 +122,18 @@ class StaticFileSet(EditablePackageSet): except KeyError: raise SetConfigError("Could not find repository '%s'" % match.groupdict()["reponame"]) if os.path.isdir(directory): - for filename in os.listdir(directory): - if filename.endswith(".metadata"): - continue - myname = name_pattern.replace("$name", filename) - myname = myname.replace("${name}", filename) - rValue[myname] = StaticFileSet(os.path.join(directory, filename), greedy=greedy, dbapi=trees["vartree"].dbapi) + directory = normalize_path(directory) + for parent, dirs, files in os.walk(directory): + for filename in files: + if filename.endswith(".metadata"): + continue + filename = os.path.join(parent, + filename)[1 + len(directory):] + myname = name_pattern.replace("$name", filename) + myname = myname.replace("${name}", filename) + rValue[myname] = StaticFileSet( + os.path.join(directory, filename), + greedy=greedy, dbapi=trees["vartree"].dbapi) return rValue multiBuilder = classmethod(multiBuilder) diff --git a/pym/portage/tests/dep/test_src_uri.py b/pym/portage/tests/dep/test_src_uri.py new file mode 100644 index 000000000..84c9f29b6 --- /dev/null +++ b/pym/portage/tests/dep/test_src_uri.py @@ -0,0 +1,34 @@ +# Copyright 2008 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Id$ + +from portage.tests import TestCase +from portage.dep import paren_reduce +from portage.dbapi.porttree import _src_uri_validate +from portage.exception import InvalidDependString + +class SrcUri(TestCase): + + def testSrcUri(self): + + tests = [ + ( "0", "http://foo/bar -> blah.tbz2" , False ), + ( "1", "http://foo/bar -> blah.tbz2" , False ), + ( "2", "http://foo/bar -> blah.tbz2" , True ), + ( "2", "foo? ( http://foo/bar -> blah.tbz2 )" , True ), + ( "2", "http://foo/bar -> foo? ( ftp://foo/a )" , False ), + ( "2", "http://foo/bar -> bar.tbz2 foo? ( ftp://foo/a )" , True ), + ( "2", "http://foo/bar blah.tbz2 ->" , False ), + ( "2", "-> http://foo/bar blah.tbz2 )" , False ), + ( "2", "http://foo/bar ->" , False ), + ( "2", "foo? ( http://foo/bar -> ) blah.tbz2" , False ), + ( "2", "http://foo/bar -> foo/blah.tbz2" , False ), + ] + + for eapi, src_uri, valid in tests: + try: + _src_uri_validate("cat/pkg-1", eapi, paren_reduce(src_uri)) + except InvalidDependString: + self.assertEqual(valid, False) + else: + self.assertEqual(valid, True)