From cf0d54f2bae0688ccdfc86f3d60735a0c3e05cf6 Mon Sep 17 00:00:00 2001 From: Fabian Groffen Date: Sat, 15 Nov 2008 16:20:49 +0000 Subject: [PATCH] Merged from trunk -r11845:11860 | 11846 | Inside depgraph.loadResumeCommand(), initialize | | zmedico | self._scheduler_graph when in --nodeps mode in order to | | | avoid an AttributeError later when self.schedulerGraph() is | | | called. | | 11847 | When in --nodeps mode, make Scheduler._set_digraph() discard | | zmedico | the digraph and make Scheduler._choose_pkg() always return | | | the task at the front of the queue. | | 11848 | Fix logic from previous commit in Scheduler._choose_pkg() to | | zmedico | ensure that the --nodeps code is only triggered when --jobs | | | > 1. | | 11849 | Add git support. Thanks to Daniel Robbins for the initial | | zmedico | patch. | | 11850 | Fix pretend output to show git -a option when committing | | zmedico | signed manifest. | | 11851 | Update hardcoded "cvs" error messages to show the correct | | zmedico | vcs value. | | 11852 | Inside have_profile_dir(), check for existence of | | zmedico | profiles.desc since that makes a little more sense thatn | | | checking for package.mask. | | 11853 | Make FindPortdir() return (None, None, None) on failure, | | zmedico | instead of raising a potentially ambiguous ValueError. | | 11854 | Derive the changelog_path variable from the new | | zmedico | checkdir_relative variable. | | 11855 | Make FindPortdir() fall back to have_profile_dir() checks if | | zmedico | it can't match the current location with anything from | | | PORTDIR_OVERLAY. Assume that an overlay will contain at | | | least a "repo_name" file while a master repo (portdir) will | | | contain at least a "profiles.desc" file. | | 11856 | Simplify code for cloning config instances inside | | zmedico | Scheduler.merge(). | | 11858 | Allow --keep-going to continue in some cases when a runtime | | zmedico | dependency has failed to build or install. This involves | | | pruning off the parts of the graph containing installed | | | packages with unsatisfied dependencies. | | 11859 | Make post_emerge() exit early if it detects that the vdb | | zmedico | state hasn't changed. This works by comparing a hash of the | | | COUNTER values for all packages in the vdb. | svn path=/main/branches/prefix/; revision=11943 --- bin/repoman | 101 +++++++++++++++++++++++++++-------- pym/_emerge/__init__.py | 31 ++++++++--- pym/portage/__init__.py | 1 + pym/portage/dbapi/vartree.py | 16 ++++++ pym/repoman/utilities.py | 28 +++++++--- 5 files changed, 143 insertions(+), 34 deletions(-) diff --git a/bin/repoman b/bin/repoman index bf2b6a7b0..f63c724aa 100755 --- a/bin/repoman +++ b/bin/repoman @@ -69,6 +69,7 @@ from portage.process import find_binary, spawn from portage.output import bold, create_color_func, darkgreen, \ green, nocolor, red, turquoise, yellow from portage.output import ConsoleStyleFile, StyleWriter +from portage.util import writemsg_level util.initialize_logger() @@ -480,12 +481,17 @@ if options.mode in ('last', 'lfull'): # commit (like if Manifest generation fails). can_force = True +portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings) +if portdir is None: + sys.exit(1) vcs = None if os.path.isdir("CVS"): vcs = "cvs" if os.path.isdir(".svn"): vcs = "svn" +elif os.path.isdir(os.path.join(portdir_overlay, ".git")): + vcs = "git" if vcs == "cvs" and \ "commit" == options.mode and \ @@ -509,11 +515,6 @@ if options.mode == 'commit' and not options.pretend and not vcs: logging.info("Not in a version controlled repository; enabling pretend mode.") options.pretend = True -try: - portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings) -except ValueError: - sys.exit(1) - os.environ["PORTDIR"] = portdir if portdir_overlay != portdir: os.environ["PORTDIR_OVERLAY"] = portdir_overlay @@ -749,17 +750,24 @@ modified_changelogs = set() mychanged = [] mynew = [] myremoved = [] +path_lstrip_re = re.compile(r'.*/') if vcs == "cvs": mycvstree = cvstree.getentries("./", recursive=1) mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./") mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./") - if vcs == "svn": svnstatus = os.popen("svn status").readlines() mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ] mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ] - +elif vcs == "git": + mychanged = os.popen("git ls-files -m").readlines() + mychanged = [ "./" + elem[:-1] for elem in mychanged ] + mynew = os.popen("git diff --cached --name-only --diff-filter=A").readlines() + strip_levels = repolevel - 1 + if strip_levels: + mynew = [path_lstrip_re.sub("", elem, strip_levels) for elem in mynew] + mynew = ["./" + elem[:-1] for elem in mynew] if vcs: new_ebuilds.update(x for x in mynew if x.endswith(".ebuild")) modified_changelogs.update(x for x in chain(mychanged, mynew) \ @@ -786,6 +794,12 @@ for x in scanlist: eadded=[] catdir,pkgdir=x.split("/") checkdir=repodir+"/"+x + checkdir_relative = "" + if repolevel < 3: + checkdir_relative = os.path.join(pkgdir, checkdir_relative) + if repolevel < 2: + checkdir_relative = os.path.join(catdir, checkdir_relative) + checkdir_relative = os.path.join(".", checkdir_relative) if options.mode == "manifest" or \ options.mode in ('commit', 'fix') and not options.pretend: @@ -871,7 +885,17 @@ for x in scanlist: s = s[s.rfind("\n") + 1:] fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s)) - if vcs and check_ebuild_notadded: + if vcs == "git" and check_ebuild_notadded: + myf = os.popen("git ls-files --others %s" % \ + (portage._shell_quote(checkdir_relative),)) + for l in myf: + if l[:-1][-7:] == ".ebuild": + stats["ebuild.notadded"] += 1 + fails["ebuild.notadded"].append( + os.path.join(x, os.path.basename(l[:-1]))) + myf.close() + + if vcs in ("cvs", "svn") and check_ebuild_notadded: try: if vcs == "cvs": myf=open(checkdir+"/CVS/Entries","r") @@ -1042,12 +1066,7 @@ for x in scanlist: del metadata_bad - changelog_path = "ChangeLog" - if repolevel < 3: - changelog_path = os.path.join(pkgdir, changelog_path) - if repolevel < 2: - changelog_path = os.path.join(catdir, changelog_path) - changelog_path = os.path.join(".", changelog_path) + changelog_path = os.path.join(checkdir_relative, "ChangeLog") changelog_modified = changelog_path in modified_changelogs allmasked = True @@ -1068,11 +1087,10 @@ for x in scanlist: if stat.S_IMODE(os.stat(full_path).st_mode) & 0111: stats["file.executable"] += 1 fails["file.executable"].append(x+"/"+y+".ebuild") - if vcs and check_ebuild_notadded and y not in eadded: + if vcs in ("cvs", "svn") and check_ebuild_notadded and y not in eadded: #ebuild not added to vcs stats["ebuild.notadded"]=stats["ebuild.notadded"]+1 fails["ebuild.notadded"].append(x+"/"+y+".ebuild") - myesplit=portage.pkgsplit(y) if myesplit is None or myesplit[0] != x.split("/")[-1]: stats["ebuild.invalidname"]=stats["ebuild.invalidname"]+1 @@ -1648,7 +1666,6 @@ else: raise # TODO propogate this except: err("Error retrieving CVS tree; exiting.") - if vcs == "svn": try: svnstatus=os.popen("svn status --no-ignore").readlines() @@ -1657,6 +1674,12 @@ else: raise # TODO propogate this except: err("Error retrieving SVN info; exiting.") + if vcs == "git": + # get list of files not under version control or missing + myf = os.popen("git ls-files --others") + myunadded = [ "./" + elem[:-1] for elem in myf ] + myf.close() + myautoadd=[] if myunadded: for x in range(len(myunadded)-1,-1,-1): @@ -1676,14 +1699,19 @@ else: print "(cvs add "+" ".join(myautoadd)+")" if vcs == "svn": print "(svn add "+" ".join(myautoadd)+")" + elif vcs == "git": + print "(git add "+" ".join(myautoadd)+")" retval=0 else: if vcs == "cvs": retval=os.system("cvs add "+" ".join(myautoadd)) if vcs == "svn": retval=os.system("svn add "+" ".join(myautoadd)) + elif vcs == "git": + retval=os.system("git add "+" ".join(myautoadd)) if retval: - print "!!! Exiting on vcs (shell) error code:",retval + writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \ + (vcs, retval), level=logging.ERROR, noiselevel=-1) sys.exit(retval) if myunadded: @@ -1724,6 +1752,19 @@ else: expansion = set("./" + prop.split(" - ")[0] \ for prop in props if " - " in prop) + elif vcs == "git": + mychanged = os.popen("git ls-files -m").readlines() + mychanged = [ "./" + elem[:-1] for elem in mychanged ] + mynew = os.popen("git diff --cached --name-only --diff-filter=A").readlines() + strip_levels = repolevel - 1 + if strip_levels: + mynew = [path_lstrip_re.sub("", elem, strip_levels) for elem in mynew] + mynew = ["./" + elem[:-1] for elem in mynew] + myremoved = os.popen("git diff --cached --name-only --diff-filter=D").readlines() + if strip_levels: + myremoved = [path_lstrip_re.sub("", elem, strip_levels) for elem in myremoved] + myremoved = ["./" + elem[:-1] for elem in myremoved] + if vcs: if not (mychanged or mynew or myremoved): print green("RepoMan sez:"), "\"Doing nothing is not always good for QA.\"" @@ -1845,6 +1886,9 @@ else: if vcs == "svn": print "(svn commit -F %s %s)" % \ (commitmessagefile, " ".join(myfiles)) + elif vcs == "git": + print "(git commit -F %s %s)" % \ + (commitmessagefile, " ".join(myfiles)) else: if vcs == "cvs": retval = spawn(["cvs", "-q", "commit", @@ -1854,12 +1898,17 @@ else: retval = spawn(["svn", "commit", "-F", commitmessagefile] + myfiles, env=os.environ) + elif vcs == "git": + retval = spawn(["git", "commit", "-F", + commitmessagefile] + myfiles, + env=os.environ) try: os.unlink(commitmessagefile) except OSError: pass if retval: - print "!!! Exiting on cvs (shell) error code:",retval + writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \ + (vcs, retval), level=logging.ERROR, noiselevel=-1) sys.exit(retval) # Setup the GPG commands @@ -1963,6 +2012,8 @@ else: print "(cvs -q commit -F commitmessagefile)" if vcs == "svn": print "(svn -q commit -F commitmessagefile)" + elif vcs == "git": + print "(git commit -F commitmessagefile)" else: fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") mymsg = os.fdopen(fd, "w") @@ -1973,12 +2024,15 @@ else: retval=os.system("cvs -q commit -F "+commitmessagefile) if vcs == "svn": retval=os.system("svn -q commit -F "+commitmessagefile) + elif vcs == "git": + retval=os.system("git commit -F "+commitmessagefile) try: os.unlink(commitmessagefile) except OSError: pass if retval: - print "!!! Exiting on cvs (shell) error code:",retval + writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \ + (vcs, retval), level=logging.ERROR, noiselevel=-1) sys.exit(retval) manifest_commit_required = False @@ -2032,6 +2086,8 @@ else: print "(cvs -q commit -F commitmessagefile)" if vcs == "svn": print "(svn -q commit -F commitmessagefile)" + elif vcs == "git": + print "(git commit -a -F commitmessagefile)" else: fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") mymsg = os.fdopen(fd, "w") @@ -2045,12 +2101,15 @@ else: retval=os.system("cvs -q commit -F "+commitmessagefile) if vcs == "svn": retval=os.system("svn -q commit -F "+commitmessagefile) + elif vcs == "git": + retval=os.system("git commit -a -F "+commitmessagefile) try: os.unlink(commitmessagefile) except OSError: pass if retval: - print "!!! Exiting on cvs (shell) error code:",retval + writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \ + (vcs, retval), level=logging.ERROR, noiselevel=-1) sys.exit(retval) print diff --git a/pym/_emerge/__init__.py b/pym/_emerge/__init__.py index 7ab37e4c9..d2c9470f5 100644 --- a/pym/_emerge/__init__.py +++ b/pym/_emerge/__init__.py @@ -7885,6 +7885,7 @@ class depgraph(object): if not serialized_tasks or "--nodeps" in self.myopts: self._serialized_tasks_cache = serialized_tasks + self._scheduler_graph = self.digraph else: self._select_package = self._select_pkg_from_graph self.myparams.add("selective") @@ -9309,8 +9310,8 @@ class Scheduler(PollScheduler): return interactive_tasks def _set_digraph(self, digraph): - if self._max_jobs is not True and \ - self._max_jobs < 2: + if "--nodeps" in self.myopts or \ + (self._max_jobs is not True and self._max_jobs < 2): # save some memory self._digraph = None return @@ -9668,7 +9669,7 @@ class Scheduler(PollScheduler): root_config.settings.lock() self.pkgsettings[root] = portage.config( - clone=self.trees[root]["vartree"].settings) + clone=root_config.settings) rval = self._check_manifests() if rval != os.EX_OK: @@ -9949,7 +9950,9 @@ class Scheduler(PollScheduler): return None if self._digraph is None: - if self._jobs or self._task_queues.merge: + if (self._jobs or self._task_queues.merge) and \ + not ("--nodeps" in self.myopts and \ + (self._max_jobs is True or self._max_jobs > 1)): self._choose_pkg_return_early = True return None return self._pkg_queue.pop(0) @@ -11275,6 +11278,12 @@ def post_emerge(root_config, myopts, mtimedb, retval): _flush_elog_mod_echo() + counter_hash = settings.get("PORTAGE_COUNTER_HASH") + if counter_hash is not None and \ + counter_hash == vardbapi._counter_hash(): + # If vdb state has not changed then there's nothing else to do. + sys.exit(retval) + vdb_path = os.path.join(target_root, portage.VDB_PATH) portage.util.ensure_dirs(vdb_path) vdb_lock = None @@ -12890,7 +12899,7 @@ def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner, # dependency to become unsatisfied. for parent_node in graph.parent_nodes(pkg): if not isinstance(parent_node, Package) \ - or parent_node.operation != "merge": + or parent_node.operation not in ("merge", "nomerge"): continue unsatisfied = \ graph.child_nodes(parent_node, @@ -12909,7 +12918,14 @@ def resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner, # it's already installed, but it has unsatisfied PDEPEND. raise mergelist[:] = pruned_mergelist - dropped_tasks.update(unsatisfied_parents) + + # Exclude installed packages that have been removed from the graph due + # to failure to build/install runtime dependencies after the dependent + # package has already been installed. + dropped_tasks.update(pkg for pkg in \ + unsatisfied_parents if pkg.operation != "nomerge") + mydepgraph.break_refs(unsatisfied_parents) + del e, graph, traversed_nodes, \ unsatisfied_parents, unsatisfied_stack continue @@ -13910,6 +13926,9 @@ def emerge_main(): mysettings = trees[myroot]["vartree"].settings mysettings.unlock() adjust_config(myopts, mysettings) + mysettings["PORTAGE_COUNTER_HASH"] = \ + trees[myroot]["vartree"].dbapi._counter_hash() + mysettings.backup_changes("PORTAGE_COUNTER_HASH") mysettings.lock() del myroot, mysettings diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index b7aa7113b..b09acaf70 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -1009,6 +1009,7 @@ class config(object): "GENTOO_MIRRORS", "NOCONFMEM", "O", "PORTAGE_BACKGROUND", "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_CALLER", + "PORTAGE_COUNTER_HASH", "PORTAGE_ECLASS_WARNING_ENABLE", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 436739459..99d27f263 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1298,6 +1298,21 @@ class vardbapi(dbapi): level=logging.ERROR, noiselevel=-1) return 0 + def _counter_hash(self): + try: + from hashlib import md5 as new_hash + except ImportError: + from md5 import new as new_hash + h = new_hash() + aux_keys = ["COUNTER"] + for cpv in self.cpv_all(): + try: + counter, = self.aux_get(cpv, aux_keys) + except KeyError: + continue + h.update(counter) + return h.hexdigest() + def cpv_inject(self, mycpv): "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" os.makedirs(self.getpath(mycpv)) @@ -1441,6 +1456,7 @@ class vardbapi(dbapi): self.invalidentry(self.getpath(subpath)) continue returnme.append(subpath) + returnme.sort() return returnme def cp_all(self, use_cache=1): diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index 10f267272..cee52186e 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -95,11 +95,13 @@ def detect_vcs_conflicts(options, vcs): sys.exit(retval) -def have_profile_dir(path, maxdepth=3): - """ Try to figure out if 'path' has a /profiles dir in it by checking for a package.mask file +def have_profile_dir(path, maxdepth=3, filename="profiles.desc"): + """ + Try to figure out if 'path' has a profiles/ + dir in it by checking for the given filename. """ while path != "/" and maxdepth: - if os.path.exists(path + "/profiles/package.mask"): + if os.path.exists(os.path.join(path, "profiles", filename)): return normalize_path(path) path = normalize_path(path + "/..") maxdepth -= 1 @@ -379,9 +381,21 @@ def FindPortdir(settings): if have_profile_dir(location, subdir.count("/")): portdir = portdir_overlay break - - del p, s, path_ids - + + # Couldn't match location with anything from PORTDIR_OVERLAY, + # so fall back to have_profile_dir() checks alone. Assume that + # an overlay will contain at least a "repo_name" file while a + # master repo (portdir) will contain at least a "profiles.desc" + # file. + if not portdir_overlay: + portdir_overlay = have_profile_dir(location, filename="repo_name") + if portdir_overlay: + subdir = location[len(portdir_overlay):] + if subdir and subdir[-1] != os.sep: + subdir += os.sep + if have_profile_dir(location, subdir.count(os.sep)): + portdir = portdir_overlay + if not portdir_overlay: if (settings["PORTDIR"] + os.path.sep).startswith(location): portdir_overlay = settings["PORTDIR"] @@ -393,7 +407,7 @@ def FindPortdir(settings): msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \ ' from the current working directory' logging.critical(msg) - raise ValueError(msg) + return (None, None, None) if not portdir: portdir = settings["PORTDIR"] -- 2.26.2