From 06f47b2d2547c3119f30d5971e5d7be89dbe26be Mon Sep 17 00:00:00 2001
From: Fabian Groffen <grobian@gentoo.org>
Date: Sat, 3 Feb 2007 21:52:00 +0000
Subject: [PATCH] Merged from trunk 5861:5880

fix doc typo; more test fiddling (WarnerBro)

A somewhat more intelligent layout, move tests into portaage namespace
so we can import them easier, seems to be a more common layout from what
I've seen.  Also rewrite the test runner to work with new layout
(WarnerBro)

apparently I'm retarded and missed some of the tests.. (WarnerBro)

Rename dir.bz2 since that's the most likely case now.

add warning about storing data in PORTDIR (WarnerBro)

For bug #153109, replace our old chflags module with the one provided by
the py-freebsd package.  Thanks to Diego Petten?\195?\178
<flameeyes@gentoo.org> for the initial patch.

remove redundant directory now (WarnerBro)

grrr I thought I removed this (WarnerBro)

For bug #165065, use the proper RESTRICT setting when fetching files in
digestgen().

Make the tests pass.

For bug #165034, make sure that symlinks are excluded from chmod calls.

Use ! -type l to exclude symlinks better.  Thanks to marienz.

Fix a typo from the previos commit.

In order to correctly handl the case of target caches that don't have
complete eclass entries, validate the set of interited eclasses to make
sure it is correct (since timestamp validation is not applicable).


svn path=/main/branches/prefix/; revision=5885
---
 bin/ebuild.sh                                 |  2 +-
 bin/emerge                                    |  2 +-
 man/make.conf.5                               |  9 +++
 pym/portage/__init__.py                       | 69 +++++++++----------
 pym/portage/cache/util.py                     |  4 +-
 pym/portage/dep.py                            |  4 +-
 {tests => pym/portage/tests}/__init__.py      | 10 ++-
 .../portage/tests}/dep/__init__.py            |  0
 .../portage/tests}/dep/test_dep_getcpv.py     |  0
 .../portage/tests}/dep/test_dep_getslot.py    |  0
 .../portage/tests}/dep/test_dep_getusedeps.py | 16 ++---
 .../portage/tests}/dep/test_get_operator.py   |  0
 .../portage/tests}/dep/test_isjustname.py     |  0
 .../portage/tests}/dep/test_isvalidatom.py    |  2 +-
 .../tests}/dep/test_match_from_list.py        |  0
 .../portage/tests}/news/__init__.py           |  0
 .../portage/tests}/news/test_NewsItem.py      |  0
 {tests => pym/portage/tests}/runTests         |  6 +-
 .../portage/tests}/util/__init__.py           |  0
 .../portage/tests}/util/test_grabdict.py      |  0
 .../tests}/util/test_normalizedPath.py        |  0
 .../portage/tests}/util/test_stackDictList.py |  0
 .../portage/tests}/util/test_stackDicts.py    |  0
 .../portage/tests}/util/test_stackLists.py    |  0
 .../portage/tests}/util/test_uniqueArray.py   |  0
 .../portage/tests}/util/test_varExpand.py     |  0
 .../portage/tests}/versions/__init__.py       |  0
 .../portage/tests}/versions/test_vercmp.py    |  0
 tests/portage/__init__.py                     |  4 --
 29 files changed, 67 insertions(+), 61 deletions(-)
 rename {tests => pym/portage/tests}/__init__.py (75%)
 rename {tests/portage => pym/portage/tests}/dep/__init__.py (100%)
 rename {tests/portage => pym/portage/tests}/dep/test_dep_getcpv.py (100%)
 rename {tests/portage => pym/portage/tests}/dep/test_dep_getslot.py (100%)
 rename {tests/portage => pym/portage/tests}/dep/test_dep_getusedeps.py (74%)
 rename {tests/portage => pym/portage/tests}/dep/test_get_operator.py (100%)
 rename {tests/portage => pym/portage/tests}/dep/test_isjustname.py (100%)
 rename {tests/portage => pym/portage/tests}/dep/test_isvalidatom.py (96%)
 rename {tests/portage => pym/portage/tests}/dep/test_match_from_list.py (100%)
 rename {tests/portage => pym/portage/tests}/news/__init__.py (100%)
 rename {tests/portage => pym/portage/tests}/news/test_NewsItem.py (100%)
 rename {tests => pym/portage/tests}/runTests (69%)
 rename {tests/portage => pym/portage/tests}/util/__init__.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_grabdict.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_normalizedPath.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_stackDictList.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_stackDicts.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_stackLists.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_uniqueArray.py (100%)
 rename {tests/portage => pym/portage/tests}/util/test_varExpand.py (100%)
 rename {tests/portage => pym/portage/tests}/versions/__init__.py (100%)
 rename {tests/portage => pym/portage/tests}/versions/test_vercmp.py (100%)
 delete mode 100644 tests/portage/__init__.py

diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 8fcf28db2..8138f99e8 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -608,7 +608,7 @@ unpack() {
 	done
 	# Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE
 	# should be preserved.
-	find . -mindepth 1 -print0 | ${XARGS} -0 chmod -f a+rX,u+w,g-w,o-w
+	find . -mindepth 1 ! -type l -print0 | ${XARGS} -0 chmod -f a+rX,u+w,g-w,o-w
 }
 
 strip_duplicate_slashes() {
diff --git a/bin/emerge b/bin/emerge
index 7194ea3d0..0b7ef0328 100755
--- a/bin/emerge
+++ b/bin/emerge
@@ -3471,7 +3471,7 @@ def chk_updated_info_files(root, infodirs, prev_mtimes, retval):
 			for inforoot in regen_infodirs:
 				if inforoot=='':
 					continue
-				for filename in ("dir", "dir.gz"):
+				for filename in ("dir", "dir.gz", "dir.bz2"):
 					file_path = os.path.join(inforoot, filename)
 					try:
 						os.rename(file_path, file_path + ".old")
diff --git a/man/make.conf.5 b/man/make.conf.5
index 85912d9bd..8e89cca75 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -381,6 +381,15 @@ Defaults to /var/tmp.
 Defines the location of your Portage tree.
 .br
 Defaults to /usr/portage. 
+.br
+\fB***Warning***\fR
+.br
+Storing data such as distfiles and binary packages in PORTDIR can
+be potentially dangerous. The default emerge configuration defaults to this 
+behavior (which will work fine), however if you deviate from the default
+configuration by changing PKGDIR or DISTDIR, emerge \-\-sync may inadvertedly
+delete all your distfiles/binary packages. Please make sure to set
+\fBPORTAGE_EXTRA_RSYNC_OPTS\fR properly to insure the safety of files in PORTDIR.
 .TP
 \fBPORTDIR_OVERLAY\fR = \fI"[path] [different\-path] [etc...]"\fR
 Defines the directories in which user made ebuilds may be stored and not 
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py
index d4f7740ac..a91fccff0 100644
--- a/pym/portage/__init__.py
+++ b/pym/portage/__init__.py
@@ -42,12 +42,12 @@ except ImportError, e:
 	sys.stderr.write("    "+str(e)+"\n\n");
 	raise
 
-try:
-	# XXX: This should get renamed to bsd_chflags, I think.
-	import chflags
-	bsd_chflags = chflags
-except ImportError:
-	bsd_chflags = None
+bsd_chflags = None
+if os.uname()[0] in ["FreeBSD"]:
+	try:
+		import freebsd as bsd_chflags
+	except ImportError:
+		pass
 
 try:
 	from portage.cache.cache_errors import CacheError
@@ -2736,19 +2736,29 @@ def digestgen(myarchives, mysettings, overwrite=1, manifestonly=0, myportdb=None
 			if missing_files:
 				mytree = os.path.realpath(os.path.dirname(
 					os.path.dirname(mysettings["O"])))
-				myuris = []
+				fetch_settings = config(clone=mysettings)
+				debug = mysettings.get("PORTAGE_DEBUG") == "1"
 				for myfile in missing_files:
+					success = False
 					for cpv in distfiles_map[myfile]:
+						myebuild = os.path.join(mysettings["O"],
+							catsplit(cpv)[1] + ".ebuild")
+						# for RESTRICT=fetch, mirror, etc...
+						doebuild_environment(myebuild, "fetch",
+							mysettings["ROOT"], fetch_settings,
+							debug, 1, myportdb)
 						alluris, aalist = myportdb.getfetchlist(
 							cpv, mytree=mytree, all=True,
-							mysettings=mysettings)
-						for uri in alluris:
-							if os.path.basename(uri) == myfile:
-								myuris.append(uri)
-				if not fetch(myuris, mysettings):
-					writemsg(("!!! File %s doesn't exist, can't update " + \
-						"Manifest\n") % myfile, noiselevel=-1)
-					return 0
+							mysettings=fetch_settings)
+						myuris = [uri for uri in alluris \
+							if os.path.basename(uri) == myfile]
+						if fetch(myuris, fetch_settings):
+							success = True
+							break
+					if not success:
+						writemsg(("!!! File %s doesn't exist, can't update " + \
+							"Manifest\n") % myfile, noiselevel=-1)
+						return 0
 		writemsg_stdout(">>> Creating Manifest for %s\n" % mysettings["O"])
 		try:
 			mf.create(requiredDistfiles=myarchives,
@@ -3777,24 +3787,12 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):
 		destexists=0
 
 	if bsd_chflags:
-		# Check that we can actually unset schg etc flags...
-		# Clear the flags on source and destination; we'll reinstate them after merging
 		if destexists and dstat.st_flags != 0:
-			if bsd_chflags.lchflags(dest, 0) < 0:
-				writemsg("!!! Couldn't clear flags on file being merged: \n ",
-					noiselevel=-1)
-		# We might have an immutable flag on the parent dir; save and clear.
-		pflags=bsd_chflags.lgetflags(os.path.dirname(dest))
+			bsd_chflags.lchflags(dest, 0)
+		pflags = os.stat(os.path.dirname(dest)).st_flags
 		if pflags != 0:
 			bsd_chflags.lchflags(os.path.dirname(dest), 0)
 
-		if (destexists and bsd_chflags.lhasproblems(dest) > 0) or \
-			bsd_chflags.lhasproblems(os.path.dirname(dest)) > 0:
-			# This is bad: we can't merge the file with these flags set.
-			writemsg("!!! Can't merge file "+dest+" because of flags set\n",
-				noiselevel=-1)
-			return None
-
 	if destexists:
 		if stat.S_ISLNK(dstat[stat.ST_MODE]):
 			try:
@@ -3897,10 +3895,8 @@ def movefile(src,dest,newmtime=None,sstat=None,mysettings=None):
 
 	if bsd_chflags:
 		# Restore the flags we saved before moving
-		if pflags and bsd_chflags.lchflags(os.path.dirname(dest), pflags) < 0:
-			writemsg("!!! Couldn't restore flags (%s) on '%s'\n" % \
-				(str(pflags), os.path.dirname(dest)), noiselevel=-1)
-			return None
+		if pflags:
+			bsd_chflags.lchflags(os.path.dirname(dest), pflags)
 
 	return newmtime
 
@@ -7625,10 +7621,9 @@ class dblink:
 
 					if bsd_chflags:
 						# Save then clear flags on dest.
-						dflags=bsd_chflags.lgetflags(mydest)
-						if dflags != 0 and bsd_chflags.lchflags(mydest, 0) < 0:
-							writemsg("!!! Couldn't clear flags on '"+mydest+"'.\n",
-								noiselevel=-1)
+						dflags = os.lstat(mydest).st_flags
+						if dflags != 0:
+							bsd_chflags.lchflags(mydest, 0)
 
 					if not os.access(mydest, os.W_OK):
 						pkgstuff = pkgsplit(self.pkg)
diff --git a/pym/portage/cache/util.py b/pym/portage/cache/util.py
index 0fc56b5ee..c31dc9163 100644
--- a/pym/portage/cache/util.py
+++ b/pym/portage/cache/util.py
@@ -39,7 +39,9 @@ def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None,
 		trg = None
 		try:
 			trg = trg_cache[x]
-			if long(trg["_mtime_"]) == long(entry["_mtime_"]) and portage.eclass_cache.is_eclass_data_valid(trg["_eclasses_"]):
+			if long(trg["_mtime_"]) == long(entry["_mtime_"]) and \
+				eclass_cache.is_eclass_data_valid(trg["_eclasses_"]) and \
+				set(trg["_eclasses_"]) == set(entry["_eclasses_"]):
 				write_it = False
 		except (cache_errors.CacheError, KeyError):
 			pass
diff --git a/pym/portage/dep.py b/pym/portage/dep.py
index e59715f1c..bf8c0ae05 100644
--- a/pym/portage/dep.py
+++ b/pym/portage/dep.py
@@ -18,7 +18,7 @@
 # "a? ( b? ( z ) ) -- Valid
 #
 
-import re,  sys, types
+import re, sys, types
 import portage.exception
 from portage.exception import InvalidData, InvalidAtom
 from portage.versions import catpkgsplit, catsplit, pkgcmp, pkgsplit, ververify
@@ -341,7 +341,7 @@ def dep_getusedeps( depend ):
 	Pull a listing of USE Dependencies out of a dep atom.
 	
 	Example usage:
-		>>> dep_getusedeps('app-misc/test:3[foo][-bar]
+		>>> dep_getusedeps('app-misc/test:3[foo][-bar]')
 		['foo','-bar']
 	
 	@param depend: The depstring to process
diff --git a/tests/__init__.py b/pym/portage/tests/__init__.py
similarity index 75%
rename from tests/__init__.py
rename to pym/portage/tests/__init__.py
index f9bca5ee6..3a881348a 100644
--- a/tests/__init__.py
+++ b/pym/portage/tests/__init__.py
@@ -7,7 +7,7 @@ import os, unittest
 
 def main():
 	
-	testDirs = ["portage", "portage/util","portage/versions", "portage/dep"]
+	testDirs = ["util","versions", "dep"]
 
 	suite = unittest.TestSuite()
 
@@ -36,7 +36,8 @@ def getTests( path, base_path ):
 	files = os.listdir( path )
 	files = [ f[:-3] for f in files if f.startswith("test_") and f.endswith(".py") ]
 	parent_path = path[len(base_path)+1:]
-	parent_module = ".".join(("tests", parent_path))
+	parent_module = ".".join(("portage","tests", parent_path))
+	parent_module = parent_module.replace('/','.')
 	result = []
 	for mymodule in files:
 		try:
@@ -47,3 +48,8 @@ def getTests( path, base_path ):
 		except ImportError:
 			raise
 	return result
+
+test_cpvs = ['sys-apps/portage','virtual/portage']
+test_versions = ['1.0', '1.0-r1','2.3_p4','1.0_alpha57']
+test_slots = [ None, '1','gentoo-sources-2.6.17','spankywashere']
+test_usedeps = ['foo','-bar', ['foo','bar'],['foo','-bar'] ]
diff --git a/tests/portage/dep/__init__.py b/pym/portage/tests/dep/__init__.py
similarity index 100%
rename from tests/portage/dep/__init__.py
rename to pym/portage/tests/dep/__init__.py
diff --git a/tests/portage/dep/test_dep_getcpv.py b/pym/portage/tests/dep/test_dep_getcpv.py
similarity index 100%
rename from tests/portage/dep/test_dep_getcpv.py
rename to pym/portage/tests/dep/test_dep_getcpv.py
diff --git a/tests/portage/dep/test_dep_getslot.py b/pym/portage/tests/dep/test_dep_getslot.py
similarity index 100%
rename from tests/portage/dep/test_dep_getslot.py
rename to pym/portage/tests/dep/test_dep_getslot.py
diff --git a/tests/portage/dep/test_dep_getusedeps.py b/pym/portage/tests/dep/test_dep_getusedeps.py
similarity index 74%
rename from tests/portage/dep/test_dep_getusedeps.py
rename to pym/portage/tests/dep/test_dep_getusedeps.py
index c16632c94..d191d43cc 100644
--- a/tests/portage/dep/test_dep_getusedeps.py
+++ b/pym/portage/tests/dep/test_dep_getusedeps.py
@@ -6,20 +6,20 @@
 from unittest import TestCase
 from portage.dep import dep_getusedeps
 
+import sys
+from portage.tests import test_cpvs, test_slots, test_versions, test_usedeps
+
 class DepGetUseDeps(TestCase):
 	""" A simple testcase for dep_getusedeps
 	"""
 
 	def testDepGetUseDeps(self):
 
-		useflags = [ '', 'foo', '-bar', ['baz','bar'], ['baz','-bar'] ]
-		cpvs = [ "sys-apps/portage" ]
-		slots = [ None, "0","1","linux-sources-2.5.7","randomstring" ]
-		versions = [ None, "2.1.1", "2.1.1-r2"]
-		for mycpv in cpvs:
-			for version in versions:
-				for slot in slots:
-					for use in useflags:
+
+		for mycpv in test_cpvs:
+			for version in test_versions:
+				for slot in test_slots:
+					for use in test_usedeps:
 						cpv = mycpv[:]
 						if version:
 							cpv += version
diff --git a/tests/portage/dep/test_get_operator.py b/pym/portage/tests/dep/test_get_operator.py
similarity index 100%
rename from tests/portage/dep/test_get_operator.py
rename to pym/portage/tests/dep/test_get_operator.py
diff --git a/tests/portage/dep/test_isjustname.py b/pym/portage/tests/dep/test_isjustname.py
similarity index 100%
rename from tests/portage/dep/test_isjustname.py
rename to pym/portage/tests/dep/test_isjustname.py
diff --git a/tests/portage/dep/test_isvalidatom.py b/pym/portage/tests/dep/test_isvalidatom.py
similarity index 96%
rename from tests/portage/dep/test_isvalidatom.py
rename to pym/portage/tests/dep/test_isvalidatom.py
index 88250e96c..2ba91b7cc 100644
--- a/tests/portage/dep/test_isvalidatom.py
+++ b/pym/portage/tests/dep/test_isvalidatom.py
@@ -20,7 +20,7 @@ class IsValidAtom(TestCase):
 			  ( ">sys-apps/portage-2.1", True ),
 			  ( "<sys-apps/portage-2.1", True ),
 			  ( "~sys-apps/portage-2.1", True ),
-			  ( "sys-apps/portage-2.1:foo", True ),
+			  ( "sys-apps/portage-2.1:foo", False ),
 			  ( "sys-apps/portage-2.1:", False ),
 			  ( ">~cate-gory/foo-1.0", False ),
 			  ( ">~category/foo-1.0", False ),
diff --git a/tests/portage/dep/test_match_from_list.py b/pym/portage/tests/dep/test_match_from_list.py
similarity index 100%
rename from tests/portage/dep/test_match_from_list.py
rename to pym/portage/tests/dep/test_match_from_list.py
diff --git a/tests/portage/news/__init__.py b/pym/portage/tests/news/__init__.py
similarity index 100%
rename from tests/portage/news/__init__.py
rename to pym/portage/tests/news/__init__.py
diff --git a/tests/portage/news/test_NewsItem.py b/pym/portage/tests/news/test_NewsItem.py
similarity index 100%
rename from tests/portage/news/test_NewsItem.py
rename to pym/portage/tests/news/test_NewsItem.py
diff --git a/tests/runTests b/pym/portage/tests/runTests
similarity index 69%
rename from tests/runTests
rename to pym/portage/tests/runTests
index c44d36788..6781796f8 100755
--- a/tests/runTests
+++ b/pym/portage/tests/runTests
@@ -10,12 +10,10 @@ import os.path as osp
 
 # Insert our parent dir so we can do shiny import "tests"
 # This line courtesy of Marienz and Pkgcore ;)
-sys.path.insert(0, osp.dirname(osp.dirname(osp.abspath(__file__))))
+sys.path.insert(0, osp.dirname(osp.dirname(osp.dirname(osp.abspath(__file__)))))
 
-# Grab SVN portage files instead of normal ones.
-sys.path.insert(0, os.path.join(sys.path[0], "pym"))
+import portage.tests as tests
 
-import tests
 if __name__ == "__main__":
 	result = tests.main()
 	if result.failures:
diff --git a/tests/portage/util/__init__.py b/pym/portage/tests/util/__init__.py
similarity index 100%
rename from tests/portage/util/__init__.py
rename to pym/portage/tests/util/__init__.py
diff --git a/tests/portage/util/test_grabdict.py b/pym/portage/tests/util/test_grabdict.py
similarity index 100%
rename from tests/portage/util/test_grabdict.py
rename to pym/portage/tests/util/test_grabdict.py
diff --git a/tests/portage/util/test_normalizedPath.py b/pym/portage/tests/util/test_normalizedPath.py
similarity index 100%
rename from tests/portage/util/test_normalizedPath.py
rename to pym/portage/tests/util/test_normalizedPath.py
diff --git a/tests/portage/util/test_stackDictList.py b/pym/portage/tests/util/test_stackDictList.py
similarity index 100%
rename from tests/portage/util/test_stackDictList.py
rename to pym/portage/tests/util/test_stackDictList.py
diff --git a/tests/portage/util/test_stackDicts.py b/pym/portage/tests/util/test_stackDicts.py
similarity index 100%
rename from tests/portage/util/test_stackDicts.py
rename to pym/portage/tests/util/test_stackDicts.py
diff --git a/tests/portage/util/test_stackLists.py b/pym/portage/tests/util/test_stackLists.py
similarity index 100%
rename from tests/portage/util/test_stackLists.py
rename to pym/portage/tests/util/test_stackLists.py
diff --git a/tests/portage/util/test_uniqueArray.py b/pym/portage/tests/util/test_uniqueArray.py
similarity index 100%
rename from tests/portage/util/test_uniqueArray.py
rename to pym/portage/tests/util/test_uniqueArray.py
diff --git a/tests/portage/util/test_varExpand.py b/pym/portage/tests/util/test_varExpand.py
similarity index 100%
rename from tests/portage/util/test_varExpand.py
rename to pym/portage/tests/util/test_varExpand.py
diff --git a/tests/portage/versions/__init__.py b/pym/portage/tests/versions/__init__.py
similarity index 100%
rename from tests/portage/versions/__init__.py
rename to pym/portage/tests/versions/__init__.py
diff --git a/tests/portage/versions/test_vercmp.py b/pym/portage/tests/versions/test_vercmp.py
similarity index 100%
rename from tests/portage/versions/test_vercmp.py
rename to pym/portage/tests/versions/test_vercmp.py
diff --git a/tests/portage/__init__.py b/tests/portage/__init__.py
deleted file mode 100644
index 920c2f7e3..000000000
--- a/tests/portage/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-# tests/portage/__init__.py -- Portage Unit Test functionality
-# Copyright 2006 Gentoo Foundation
-# Distributed under the terms of the GNU General Public License v2
-# $Id$
-- 
2.26.2