EAPI awareness, and flat_hash/flat_list autodetection for rysnc cache.
authorBrian Harring <ferringb@gentoo.org>
Sat, 24 Sep 2005 17:54:05 +0000 (17:54 -0000)
committerBrian Harring <ferringb@gentoo.org>
Sat, 24 Sep 2005 17:54:05 +0000 (17:54 -0000)
svn path=/main/branches/2.0/; revision=2022

ChangeLog
bin/ebuild.sh
pym/portage.py
pym/portage_const.py
pym/portage_db_flat.py
pym/portage_db_flat_hash.py [new file with mode: 0644]
pym/portage_db_metadata.py [new file with mode: 0644]

index 7bc4c0ba2fc894166d7f5851eb9ed9aa751657ed..953c8f25276c9530b274428d2cba5f086fc7f58f 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,6 +7,21 @@
     2. /var/cache/edb/world is now /var/lib/portage/world.
     3. /etc/portage/profile/virtuals is _USER_ configs only.
 
+  24 Sep 2005; Brian Harring <ferringb@gentoo.org> pym/portage.py,
+  +pym/portage_db_metadata.py, +pym/portage_db_flat_hash.py, 
+  pym/portage_db_flat.py, pym/portage_const.py:
+  EAPI awareness, and support for flat_hash metadata/cache.
+  Later autodetects if the rsync cache is flat_hash, or flat_list, 
+  and interprets accordingly.
+  
+  Flat_hash is intended to make it such that rsync cache breakages 
+  for older portage versions no longer occur; the container format 
+  can hold pretty much anything across EAPI versions.
+  Additionally... EAPI is intended such that additions to the ebuild 
+  env/helpers don't puke if you're running to old of a portage version; 
+  it allows us to 'version' the ebuild env/funcs, and do masking dependant
+  on that versioning.
+
   23 Sep 2005; Brian Harring <ferringb@gentoo.org> bin/emerge:
   cleanse_cache cleanups, should kill off all remaining issues 
   in bug #100444 .
index 8cba9030c2ac47946d72fce652af6a3446ec6ad3..d7311796787d245de5c6262d21d60ad941d23404 100755 (executable)
@@ -1898,6 +1898,7 @@ for myarg in $*; do
                echo `echo "$CDEPEND"`     >> $dbkey
                echo `echo "$PDEPEND"`     >> $dbkey
                echo `echo "$PROVIDE"`     >> $dbkey
+               echo `echo "${EAPI:-0}"`   >> $dbkey
                echo `echo "$UNUSED_01"`   >> $dbkey
                echo `echo "$UNUSED_02"`   >> $dbkey
                echo `echo "$UNUSED_03"`   >> $dbkey
@@ -1905,7 +1906,6 @@ for myarg in $*; do
                echo `echo "$UNUSED_05"`   >> $dbkey
                echo `echo "$UNUSED_06"`   >> $dbkey
                echo `echo "$UNUSED_07"`   >> $dbkey
-               echo `echo "$UNUSED_08"`   >> $dbkey
                set +f
                #make sure it is writable by our group:
                exit 0
index 3cfd2bad0953f586965031ce7539081e4c14243c..7ffad8dbbb79b9b8c34698b4ae627081cb746ebb 100644 (file)
@@ -81,7 +81,7 @@ try:
          MOVE_BINARY, PRELINK_BINARY, WORLD_FILE, MAKE_CONF_FILE, MAKE_DEFAULTS_FILE, \
          DEPRECATED_PROFILE_FILE, USER_VIRTUALS_FILE, EBUILD_SH_ENV_FILE, \
          INVALID_ENV_FILE, CUSTOM_MIRRORS_FILE, SANDBOX_PIDS_FILE, CONFIG_MEMORY_FILE,\
-         INCREMENTALS, STICKIES
+         INCREMENTALS, STICKIES, EAPI
 
        from portage_data import ostype, lchown, userland, secpass, uid, wheelgid, \
                                 portage_uid, portage_gid
@@ -915,7 +915,7 @@ class config:
                        if self.modules["user"] == None:
                                self.modules["user"] = {}
                        self.modules["default"] = {
-                               "portdbapi.metadbmodule": "portage_db_flat.database",
+                               "portdbapi.metadbmodule": "portage_db_metadata.database",
                                "portdbapi.auxdbmodule":  "portage_db_flat.database",
                                "eclass_cache.dbmodule":  "portage_db_cpickle.database",
                        }
@@ -2402,6 +2402,15 @@ def doebuild(myebuild,mydo,myroot,mysettings,debug=0,listonly=0,fetchonly=0,clea
                        raise
                except:
                        pass
+               try:
+                       eapi = int(db[root][tree].dbapi.aux_get(mycpv, ["EAPI"])[0])
+                       if portage_const.EAPI != eapi:
+                               # can't do anything with this.
+                               raise Exception("Unable to do any operations on '%s', due to the fact it's EAPI is higher then this portage versions.  Please upgrade to a portage version that supports EAPI %i" % (mycpv, eapi))
+               except SystemExit, e:
+                       raise
+               except Exception, e:
+                       raise Exception("Unable to pull EAPI from cpv %s, tree %s; can't confirm that it's supported by this portage, thus unable to merge it: Exception was '%s'" % (mycpv, tree, e))
 
        if mysplit[2] == "r0":
                mysettings["PVR"]=mysplit[1]
@@ -3914,7 +3923,14 @@ def getmaskingstatus(mycpv):
                                        rValue.append("package.mask")
 
        # keywords checking
-       mygroups = portdb.aux_get(mycpv, ["KEYWORDS"])[0].split()
+       mygroups, eapi = portdb.aux_get(mycpv, ["KEYWORDS", "EAPI"])
+       try:
+               eapi = abs(eapi)
+       except TypeError:
+               eapi = 1
+       if eapi != portage_const.EAPI:
+               return ["required EAPI %s, supported EAPI %s" % (eapi, portage_const.EAPI)]
+       mygroups = mygroups.split()
        pgroups=groups[:]
        myarch = settings["ARCH"]
        pkgdict = settings.pkeywordsdict
@@ -4534,7 +4550,14 @@ class bindbapi(fakedbapi):
                                else:
                                        myval = string.join(myval.split(),' ')
                                mylist.append(myval)
-
+               if "EAPI" in wants:
+                       idx = wants.index("EAPI")
+                       if mylist[idx] in ("", "0"):
+                               mylist[idx] = 0
+                       elif mylist[idx] == 0:
+                               pass
+                       else:
+                               mylist[idx] = 1
                return mylist
 
 
@@ -4814,6 +4837,14 @@ class vardbapi(dbapi):
                        else:
                                myd = ""
                        results.append(myd)
+               if "EAPI" in wants:
+                       idx = wants.index("EAPI")
+                       if mylist[idx] in ("", "0"):
+                               mylist[idx] = 0
+                       elif mylist[idx] == 0:
+                               pass
+                       else:
+                               mylist[idx] = 1
                return results
 
 
@@ -5080,9 +5111,9 @@ auxdbkeys=[
   'DEPEND',    'RDEPEND',   'SLOT',      'SRC_URI',
        'RESTRICT',  'HOMEPAGE',  'LICENSE',   'DESCRIPTION',
        'KEYWORDS',  'INHERITED', 'IUSE',      'CDEPEND',
-       'PDEPEND',   'PROVIDE',
+       'PDEPEND',   'PROVIDE', 'EAPI',
        'UNUSED_01', 'UNUSED_02', 'UNUSED_03', 'UNUSED_04',
-       'UNUSED_05', 'UNUSED_06', 'UNUSED_07', 'UNUSED_08',
+       'UNUSED_05', 'UNUSED_06', 'UNUSED_07',
        ]
 auxdbkeylen=len(auxdbkeys)
 
@@ -5296,11 +5327,26 @@ class portdbapi(dbapi):
                # we use cache files, which is usually on /usr/portage/metadata/cache/.
                if doregen and mylocation==self.mysettings["PORTDIR"] and metacachedir and self.metadb[cat].has_key(pkg):
                        metadata=self.metadb[cat][pkg]
-                       self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split())
+                       
+                       try:
+                               if metadata["EAPI"] == '0':
+                                       metadata["EAPI"] = 0
+                       except KeyError:
+                               metadata["EAPI"] = 0
+
+                       if metadata["EAPI"] != portage_const.EAPI:
+                               # intentionally wipe keys.
+                               map(lambda x: metadata.setdefault("x", ''), auxdbkeys)
+                               metadata["EAPI"] == -1
+
+                       else:
+                               # eclass updates only if we haven't nuked the entry.
+                               self.eclassdb.update_package(mylocation,cat,pkg,metadata["INHERITED"].split())
+
                        self.auxdb[mylocation][cat][pkg] = metadata
                        self.auxdb[mylocation][cat].sync()
-               elif doregen:
 
+               elif doregen:
                        writemsg("doregen: %s %s\n" % (doregen,mycpv), 2)
                        writemsg("Generating cache entry(0) for: "+str(myebuild)+"\n",1)
 
@@ -5365,6 +5411,17 @@ class portdbapi(dbapi):
                                if mylines[x][-1] == '\n':
                                        mylines[x] = mylines[x][:-1]
                                mydata[auxdbkeys[x]] = mylines[x]
+
+                       try:
+                               eapi = int(mydata["EAPI"])
+                       except ValueError:
+                               eapi = 1
+                               if eapi > portage_const.EAPI:
+                                       # if newer version, wipe everything and negate eapi
+                                       mydata = {}
+                                       map(lambda x:mydata.setdefault(x, ""), auxdbkeys)
+                                       mydata["EAPI"] = -eapi
+
                        mydata["_mtime_"] = emtime
 
                        self.auxdb[mylocation][cat][pkg] = mydata
@@ -5376,10 +5433,17 @@ class portdbapi(dbapi):
                mydata   = self.auxdb[mylocation][cat][pkg]
                returnme = []
                for x in mylist:
-                       if mydata.has_key(x):
-                               returnme.append(mydata[x])
+                       returnme.append(mydata.get(x,""))
+
+               if "EAPI" in mylist:
+                       idx = mylist.index("EAPI")
+                       if returnme[idx] in ("", "0"):
+                               returnme[idx] = 0
+                       elif returnme[idx] == 0:
+                               pass
                        else:
-                               returnme.append("")
+                               try:                            returnme[idx] = int(returnme[idx])
+                               except ValueError:      returnme[idx] = 1
 
                return returnme
 
@@ -5628,14 +5692,14 @@ class portdbapi(dbapi):
                        #we need to update this next line when we have fully integrated the new db api
                        auxerr=0
                        try:
-                               myaux=db["/"]["porttree"].dbapi.aux_get(mycpv, ["KEYWORDS"])
+                               keys, eapi = db["/"]["porttree"].dbapi.aux_get(mycpv, ["KEYWORDS", "EAPI"])
                        except (KeyError,IOError,TypeError):
                                continue
-                       if not myaux[0]:
+                       if not keys:
                                # KEYWORDS=""
                                #print "!!! No KEYWORDS for "+str(mycpv)+" -- Untested Status"
                                continue
-                       mygroups=myaux[0].split()
+                       mygroups=keys.split()
                        pgroups=groups[:]
                        match=0
                        cp = dep_getkey(mycpv)
@@ -5663,7 +5727,8 @@ class portdbapi(dbapi):
                        if not match and ((hastesting and "~*" in pgroups) or (hasstable and "*" in pgroups)):
                                match=1
                        if match:
-                               newlist.append(mycpv)
+                               if eapi == portage_const.EAPI:
+                                       newlist.append(mycpv)
                return newlist
 
 class binarytree(packagetree):
index d01f77bbb77f90e38af0c7cefd672095336a1be3..404ca3c45036238ca5cc325aa082662a14d07094 100644 (file)
@@ -43,6 +43,8 @@ CONFIG_MEMORY_FILE      = PRIVATE_PATH + "/config"
 INCREMENTALS=["USE","FEATURES","ACCEPT_KEYWORDS","ACCEPT_LICENSE","CONFIG_PROTECT_MASK","CONFIG_PROTECT","PRELINK_PATH","PRELINK_PATH_MASK"]
 STICKIES=["KEYWORDS_ACCEPT","USE","CFLAGS","CXXFLAGS","MAKEOPTS","EXTRA_ECONF","EXTRA_EINSTALL","EXTRA_EMAKE"]
 
+EAPI = 0
+
 # ===========================================================================
 # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT
 # ===========================================================================
index ddec4f591afbe935bda155ba31c8edcffae43f84..4fc220bb9e7f9de77dd0e27676554873cd19dcbd 100644 (file)
@@ -9,6 +9,10 @@ import stat
 
 import portage_db_template
 
+# since this format is massively deprecated, 
+# we're hardcoding the previously weird line count
+magic_line_count = 22
+
 class database(portage_db_template.database):
        def module_init(self):
                self.lastkey  = None # Cache
@@ -42,39 +46,39 @@ class database(portage_db_template.database):
                                mykeys += [x]
                return mykeys
 
-       def get_values(self,key):
+       def get_values(self,key, data=None):
+               """ do not use data unless you know what it does."""
+
                if not key:
                        raise KeyError, "key is not set to a valid value"
 
-               try:
-                       # give buffering a hint of the pretty much maximal cache size we deal with
-                       myf = open(self.fullpath+key, "r", 8192)
-               except OSError, oe:
-                       # either the file didn't exist, or it was removed under our feet.
-                       return None 
-               
+               mydict = {}
+               if data == None:
+                       try:
+                               # give buffering a hint of the pretty much maximal cache size we deal with
+                               myf = open(self.fullpath+key, "r", 8192)
+                       except OSError:
+                               # either the file didn't exist, or it was removed under our feet.
+                               raise KeyError("failed reading key")
+
+                       # nuke the newlines right off the batt.
+                       data = myf.read().splitlines()
+                       mydict["_mtime_"] = os.fstat(myf.fileno()).st_mtime
+                       myf.close()
+               else:
+                       mydict["_mtime_"] = data.pop(-1)
 
-               # nuke the newlines right off the batt.
-               data = myf.read().splitlines()
-               mdict = {}
-               
                # rely on exceptions to note differing line counts.
                try:
-                       for x in range(0, len(self.dbkeys)):
-                               mdict[self.dbkeys[x]] = data[x]
-
-                       # do this now, rather then earlier- possible that earlier it might have been wasted
-                       # if key count mismatched
-                       mdict["_mtime_"] = os.fstat(myf.fileno()).st_mtime
+                       for x in range(magic_line_count):
+                               mydict[self.dbkeys[x]] = data[x]
 
                except IndexError:
-                       myf.close()
                        raise ValueError, "Key count mistmatch"
 
-               myf.close()
-               return mdict
+               return mydict
        
-       def set_values(self,key,val):
+       def set_values(self,key, val, raw=False):
                if not key:
                        raise KeyError, "No key provided. key:%s val:%s" % (key,val)
                if not val:
@@ -86,12 +90,19 @@ class database(portage_db_template.database):
 
                update_fp = self.fullpath + ".update." + str(os.getpid()) + "." + key
                myf = open(update_fp,"w")
-               myf.writelines( [ val[x] +"\n" for x in self.dbkeys] )
+               if not raw:
+                       myf.writelines( [ str(val[x]) +"\n" for x in self.dbkeys] )
+                       if len(self.dbkeys) != magic_line_count:
+                               myf.writelines(["\n"] * len(self.dbkeys) - magic_line_count)
+                       mtime = val["_mtime_"]
+               else:
+                       mtime = val.pop(-1)
+                       myf.writelines(val)
                myf.close()
                
                os.chown(update_fp, self.uid, self.gid)
                os.chmod(update_fp, 0664)
-               os.utime(update_fp, (-1,long(val["_mtime_"])))
+               os.utime(update_fp, (-1,long(mtime)))
                os.rename(update_fp, self.fullpath+key)
 
        def del_key(self,key):
diff --git a/pym/portage_db_flat_hash.py b/pym/portage_db_flat_hash.py
new file mode 100644 (file)
index 0000000..8ee7661
--- /dev/null
@@ -0,0 +1,38 @@
+# Copyright 2004 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $
+cvs_id_string="$Id: portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $"[5:-2]
+
+import portage_db_flat, os
+
+class database(portage_db_flat.database):
+       
+       def get_values(self, key, data=None):
+               """ do not specify data unless you know what it does"""
+               if not key:
+                       raise KeyError("key is not valid")
+               
+               if data == None:
+                       try:
+                               myf = open(self.fullpath + key, "r")
+                       except OSError:
+                               raise KeyError("failed pulling key")
+
+                       data = dict(map(lambda x: x.split("=",1), myf.read().splitlines()))
+                       data["_mtime_"] = os.fstat(myf.fileno()).st_mtime
+                       myf.close()
+
+               mydict = {}
+               for x in self.dbkeys:
+                       mydict[x] = data.get(x, "")
+               mydict["_mtime_"] = long(data["_mtime_"])
+               return mydict
+               
+       def set_values(self, key, values):
+               l = []
+               for x in values.keys():
+                       if values[x] and x != "_mtime_":
+                               l.append("%s=%s\n" % (x, values[x]))
+               l.append(values["_mtime_"])
+               portage_db_flat.database.set_values(self, key, l, raw=True)
+               
diff --git a/pym/portage_db_metadata.py b/pym/portage_db_metadata.py
new file mode 100644 (file)
index 0000000..60d87b2
--- /dev/null
@@ -0,0 +1,49 @@
+# Copyright 2004 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+# $Header: /var/cvsroot/gentoo-src/portage/pym/Attic/portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $
+cvs_id_string="$Id: portage_db_flat.py,v 1.13.2.6 2005/04/19 07:14:17 ferringb Exp $"[5:-2]
+
+import os, portage_db_flat_hash, portage_db_flat
+
+class database(portage_db_flat_hash.database):
+       
+       def get_values(self, key):
+               if not key:
+                       raise KeyError("key is not valid")
+               
+               try:
+                       myf = open(self.fullpath + key, "r")
+               except OSError:
+                       raise KeyError("key is not valid")
+               mtime = os.fstat(myf.fileno()).st_mtime
+               data = myf.read().splitlines()
+               
+               # easy attempt first.
+               if len(data) != portage_db_flat.magic_line_count:
+                       d = dict(map(lambda x: x.split("=",1), data))
+                       d["_mtime_"] = mtime
+                       return portage_db_flat_hash.database.get_values(self, key, d)
+               # this one's interesting.
+               d = {}
+
+               for line in data:
+                       # yes, meant to iterate over a string.
+                       hashed = False
+                       for idx, c in enumerate(line):
+                               if not c.isalpha():
+                                       if c == "=" and idx > 0:
+                                               hashed = True
+                                               d[line[:idx]] = line[idx + 1:]
+                                       elif c == "_" or c.isdigit():
+                                               continue
+                                       break
+                               elif not c.isupper():
+                                       break
+
+                       if not hashed:
+                               # non hashed.
+                               data.append(mtime)
+                               return portage_db_flat.database.get_values(self, key, data=data)
+
+               d["_mtime_"] = mtime
+               return portage_db_flat_hash.database.get_values(self, key, data=d)