pym/gentoolkit/analyse/analyse.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com>
   4 # Copyright(c) 2010, Gentoo Foundation
   5 # Distributed under the terms of the GNU General Public License v2
   6 #
   7
   8 """Provides a breakdown list of USE flags or keywords used and by
   9 what packages according to the Installed package database"""
  10
  11 from __future__ import print_function
  12
  13 import sys
  14
  15 import gentoolkit
  16 from gentoolkit.dbapi import PORTDB, VARDB
  17 from gentoolkit.analyse.base import ModuleBase
  18 from gentoolkit import pprinter as pp
  19 from gentoolkit.flag import get_installed_use, get_flags
  20 from gentoolkit.analyse.lib import FlagAnalyzer, KeywordAnalyser
  21 from gentoolkit.analyse.output import nl, AnalysisPrinter
  22 from gentoolkit.package import Package
  23 from gentoolkit.helpers import get_installed_cpvs
  24
  25 import portage
  26
  27
  28 def gather_flags_info(
  29                 cpvs=None,
  30                 system_flags=None,
  31                 include_unset=False,
  32                 target="USE",
  33                 use_portage=False,
  34                 #  override-able for testing
  35                 _get_flags=get_flags,
  36                 _get_used=get_installed_use
  37                 ):
  38         """Analyse the installed pkgs USE flags for frequency of use
  39
  40         @type cpvs: list
  41         @param cpvs: optional list of [cat/pkg-ver,...] to analyse or
  42                         defaults to entire installed pkg db
  43         @type: system_flags: list
  44         @param system_flags: the current default USE flags as defined
  45                         by portage.settings["USE"].split()
  46         @type include_unset: bool
  47         @param include_unset: controls the inclusion of unset USE flags in the report.
  48         @type target: string
  49         @param target: the environment variable being analysed
  50                         one of ["USE", "PKGUSE"]
  51         @type _get_flags: function
  52         @param _get_flags: ovride-able for testing,
  53                         defaults to gentoolkit.analyse.lib.get_flags
  54         @param _get_used: ovride-able for testing,
  55                         defaults to gentoolkit.analyse.lib.get_installed_use
  56         @rtype dict. {flag:{"+":[cat/pkg-ver,...], "-":[cat/pkg-ver,...], "unset":[]}
  57         """
  58         if cpvs is None:
  59                 cpvs = VARDB.cpv_all()
  60         # pass them in to override for tests
  61         flags = FlagAnalyzer(system_flags,
  62                 filter_defaults=False,
  63                 target=target,
  64                 _get_flags=_get_flags,
  65                 _get_used=get_installed_use
  66         )
  67         flag_users = {}
  68         for cpv in cpvs:
  69                 if cpv.startswith("virtual"):
  70                         continue
  71                 if use_portage:
  72                         plus, minus, unset = flags.analyse_cpv(cpv)
  73                 else:
  74                         pkg = Package(cpv)
  75                         plus, minus, unset = flags.analyse_pkg(pkg)
  76                 for flag in plus:
  77                         if flag in flag_users:
  78                                 flag_users[flag]["+"].append(cpv)
  79                         else:
  80                                 flag_users[flag] = {"+": [cpv], "-": []}
  81                 for flag in minus:
  82                         if flag in flag_users:
  83                                 flag_users[flag]["-"].append(cpv)
  84                         else:
  85                                 flag_users[flag] = {"+":[], "-": [cpv]}
  86                 if include_unset:
  87                         for flag in unset:
  88                                 if flag in flag_users:
  89                                         if "unset" in flag_users[flag]:
  90                                                 flag_users[flag]["unset"].append(cpv)
  91                                         else:
  92                                                 flag_users[flag]["unset"] = [cpv]
  93                                 else:
  94                                         flag_users[flag] = {"+": [], "-": [], "unset": [cpv]}
  95         return flag_users
  96
  97
  98 def gather_keywords_info(
  99                 cpvs=None,
 100                 system_keywords=None,
 101                 use_portage=False,
 102                 #  override-able for testing
 103                 keywords=portage.settings["ACCEPT_KEYWORDS"],
 104                 analyser = None
 105                 ):
 106         """Analyse the installed pkgs 'keywords' for frequency of use
 107
 108         @param cpvs: optional list of [cat/pkg-ver,...] to analyse or
 109                         defaults to entire installed pkg db
 110         @param system_keywords: list of the system keywords
 111         @param keywords: user defined list of keywords to check and report on
 112                         or reports on all relevant keywords found to have been used.
 113         @param _get_kwds: overridable function for testing
 114         @param _get_used: overridable function for testing
 115         @rtype dict. {keyword:{"stable":[cat/pkg-ver,...], "testing":[cat/pkg-ver,...]}
 116         """
 117         if cpvs is None:
 118                 cpvs = VARDB.cpv_all()
 119         keyword_users = {}
 120         for cpv in cpvs:
 121                 if cpv.startswith("virtual"):
 122                         continue
 123                 if use_portage:
 124                         keyword = analyser.get_inst_keyword_cpv(cpv)
 125                 else:
 126                         pkg = Package(cpv)
 127                         keyword = analyser.get_inst_keyword_pkg(pkg)
 128                 #print "returned keyword =", cpv, keyword, keyword[0]
 129                 key = keyword[0]
 130                 if key in ["~", "-"]:
 131                         _kwd = keyword[1:]
 132                         if _kwd in keyword_users:
 133                                 if key in ["~"]:
 134                                         keyword_users[_kwd]["testing"].append(cpv)
 135                                 elif key in ["-"]:
 136                                         #print "adding cpv to missing:", cpv
 137                                         keyword_users[_kwd]["missing"].append(cpv)
 138                         else:
 139                                 if key in ["~"]:
 140                                         keyword_users[_kwd] = {"stable": [],
 141                                                 "testing": [cpv], "missing": []}
 142                                 elif key in ["-"]:
 143                                         keyword_users[_kwd] = {"stable": [],
 144                                                 "testing": [], "missing": [cpv]}
 145                                 else:
 146                                         keyword_users[_kwd] = {"stable": [cpv],
 147                                                 "testing": [], "missing": []}
 148                 elif keyword in keyword_users:
 149                                 keyword_users[keyword]["stable"].append(cpv)
 150                 else:
 151                                 keyword_users[keyword] = {
 152                                         "stable": [cpv],
 153                                         "testing": [],
 154                                         "missing": []
 155                                         }
 156         return keyword_users
 157
 158
 159 class Analyse(ModuleBase):
 160         """Installed db analysis tool to query the installed databse
 161         and produce/output stats for USE flags or keywords/mask.
 162         The 'rebuild' action output is in the form suitable for file type output
 163         to create a new package.use, package.keywords, package.unmask
 164         type files in the event of needing to rebuild the
 165         /etc/portage/* user configs
 166         """
 167         def __init__(self):
 168                 ModuleBase.__init__(self)
 169                 self.module_name = "analyse"
 170                 self.options = {
 171                         "flags": False,
 172                         "keywords": False,
 173                         "packages": False,
 174                         "unset": False,
 175                         "verbose": False,
 176                         "quiet": False,
 177                         'prefix': False,
 178                         'portage': True
 179                 }
 180                 self.module_opts = {
 181                         "-f": ("flags", "boolean", True),
 182                         "--flags": ("flags", "boolean", True),
 183                         "-k": ("keywords", "boolean", True),
 184                         "--keywords": ("keywords", "boolean", True),
 185                         "-u": ("unset", "boolean", True),
 186                         "--unset": ("unset", "boolean", True),
 187                         "-v": ("verbose", "boolean", True),
 188                         "--verbose": ("verbose", "boolean", True),
 189                         "-p": ("prefix", "boolean", True),
 190                         "--prefix": ("prefix", "boolean", True),
 191                         "-G": ("portage", "boolean", False),
 192                         "--portage": ("portage", "boolean", False),
 193                 }
 194                 self.formatted_options = [
 195                         ("  -h, --help",  "Outputs this useage message"),
 196                         ("  -a, --analyse",
 197                         "Action, sets the module to gather data and output the"),
 198                         ("", "formatted stats/information to the screen"),
 199                         ("  -u, --unset",
 200                         "Additionally include any unset USE flags and the packages"),
 201                         ("", "that could use them"),
 202                         ("  -v, --verbose",
 203                         "Used in the analyse action to output more detailed information"),
 204                         ("  -p, --prefix",
 205                         "Used for testing purposes only, runs report using " +
 206                         "a prefix keyword and 'prefix' USE flag"),
 207                         #(" -G, --portage",
 208                         #"Use portage directly instead of gentoolkit's Package " +
 209                         #"object for some operations. Usually a little faster."),
 210                 ]
 211                 self.formatted_args = [
 212                         ("  use",
 213                         "Causes the action to analyse the installed packages USE flags"),
 214                         ("  pkguse",
 215                         "Causes the action to analyse the installed packages PKGUSE flags"),
 216                         ("  ",
 217                         "These are flags that have been set in /etc/portage/package.use"),
 218                         ("  keywords",
 219                         "Causes the action to analyse the installed packages keywords"),
 220                         ("  packages",
 221                         "Causes the action to analyse the installed packages and the"),
 222                         ("  ",
 223                         "USE flags they were installed with"),
 224                 ]
 225                 self.short_opts = "huvpG"
 226                 self.long_opts = ("help", "unset", "verbose", "prefix") #, "portage")
 227                 self.need_queries = True
 228                 self.arg_spec = "Target"
 229                 self.arg_options = ['use', 'pkguse','keywords', 'packages']
 230                 self.arg_option = False
 231                 self.warning = (
 232                         "   CAUTION",
 233                         "This is beta software and some features/options are incomplete,",
 234                         "some features may change in future releases includig its name.",
 235                         "Feedback will be appreciated, http://bugs.gentoo.org")
 236
 237
 238         def run(self, input_args, quiet=False):
 239                 """runs the module
 240
 241                 @param input_args: input arguments to be parsed
 242                 """
 243                 query = self.main_setup(input_args)
 244                 query = self.validate_query(query)
 245                 if query in ["use", "pkguse"]:
 246                         self.analyse_flags(query)
 247                 elif query in ["keywords"]:
 248                         self.analyse_keywords()
 249                 elif query in ["packages"]:
 250                         self.analyse_packages()
 251
 252         def analyse_flags(self, target):
 253                 """This will scan the installed packages db and analyse the
 254                 USE flags used for installation and produce a report on how
 255                 they were used.
 256
 257                 @type target: string
 258                 @param target: the target to be analysed, one of ["use", "pkguse"]
 259                 """
 260                 system_use = portage.settings["USE"].split()
 261                 self.printer = AnalysisPrinter(
 262                                 "use",
 263                                 self.options["verbose"],
 264                                 system_use)
 265                 if self.options["verbose"]:
 266                         cpvs = VARDB.cpv_all()
 267                         #cpvs = get_installed_cpvs()
 268                         #print "Total number of installed ebuilds =", len(cpvs)
 269                         flag_users = gather_flags_info(cpvs, system_use,
 270                                 self.options["unset"], target=target.upper(),
 271                                 use_portage=self.options['portage'])
 272                 else:
 273                         cpvs = get_installed_cpvs()
 274                         flag_users = gather_flags_info(cpvs, system_flags=system_use,
 275                                 include_unset=self.options["unset"], target=target.upper(),
 276                                 use_portage=self.options['portage'])
 277                 #print flag_users
 278                 flag_keys = sorted(flag_users)
 279                 if self.options["verbose"]:
 280                         print(" Flag                              System  #pkgs   cat/pkg-ver")
 281                         blankline = nl
 282                 elif not self.options['quiet']:
 283                         print(" Flag                              System  #pkgs")
 284                         blankline = lambda: None
 285                 for flag in flag_keys:
 286                         flag_pos = flag_users[flag]["+"]
 287                         if len(flag_pos):
 288                                 self.printer(flag, "+", flag_pos)
 289                                 #blankline()
 290                         flag_neg = flag_users[flag]["-"]
 291                         if len(flag_neg):
 292                                 self.printer(flag, "-", flag_neg)
 293                                 #blankline()
 294                         if "unset" in flag_users[flag] and flag_users[flag]["unset"]:
 295                                 flag_unset = flag_users[flag]["unset"]
 296                                 self.printer(flag, "unset", flag_unset)
 297                         #blankline()
 298                 if not self.options['quiet']:
 299                         print("===================================================")
 300                         print("Total number of flags in report =",
 301                                 pp.output.red(str(len(flag_keys))))
 302                         if self.options["verbose"]:
 303                                 print("Total number of installed ebuilds =",
 304                                         pp.output.red(str(len([x for x in cpvs]))))
 305                         print()
 306
 307
 308         def analyse_keywords(self, keywords=None):
 309                 """This will scan the installed packages db and analyse the
 310                 keywords used for installation and produce a report on them.
 311                 """
 312                 print()
 313                 system_keywords = portage.settings["ACCEPT_KEYWORDS"]
 314                 arch = portage.settings["ARCH"]
 315                 if self.options["prefix"]:
 316                         # build a new keyword for testing
 317                         system_keywords = "~" + arch + "-linux"
 318                 if self.options["verbose"] or self.options["prefix"]:
 319                         print("Current system ARCH =", arch)
 320                         print("Current system ACCEPT_KEYWORDS =", system_keywords)
 321                 system_keywords = system_keywords.split()
 322                 self.printer = AnalysisPrinter(
 323                                 "keywords",
 324                                 self.options["verbose"],
 325                                 system_keywords)
 326                 self.analyser = KeywordAnalyser( arch, system_keywords, VARDB)
 327                 #self.analyser.set_order(portage.settings["USE"].split())
 328                 # only for testing
 329                 test_use = portage.settings["USE"].split()
 330                 if self.options['prefix'] and 'prefix' not in test_use:
 331                         print("ANALYSE_KEYWORDS() 'prefix' flag not found in system",
 332                                 "USE flags!!!  appending for testing")
 333                         print()
 334                         test_use.append('prefix')
 335                 self.analyser.set_order(test_use)
 336                 # /end testing
 337
 338                 if self.options["verbose"]:
 339                         cpvs = VARDB.cpv_all()
 340                         #print "Total number of installed ebuilds =", len(cpvs)
 341                         keyword_users = gather_keywords_info(
 342                                 cpvs=cpvs,
 343                                 system_keywords=system_keywords,
 344                                 use_portage=self.options['portage'],
 345                                 keywords=keywords, analyser = self.analyser
 346                                 )
 347                         blankline = nl
 348                 else:
 349                         keyword_users = gather_keywords_info(
 350                                 system_keywords=system_keywords,
 351                                 use_portage=self.options['portage'],
 352                                 keywords=keywords,
 353                                 analyser = self.analyser
 354                                 )
 355                         blankline = lambda: None
 356                 #print keyword_users
 357                 keyword_keys = sorted(keyword_users)
 358                 if self.options["verbose"]:
 359                         print(" Keyword   System  #pkgs   cat/pkg-ver")
 360                 elif not self.options['quiet']:
 361                         print(" Keyword   System  #pkgs")
 362                 for keyword in keyword_keys:
 363                         kwd_stable = keyword_users[keyword]["stable"]
 364                         if len(kwd_stable):
 365                                 self.printer(keyword, " ", kwd_stable)
 366                                 blankline()
 367                         kwd_testing = keyword_users[keyword]["testing"]
 368                         if len(kwd_testing):
 369                                 self.printer(keyword, "~", kwd_testing)
 370                                 blankline()
 371                         kwd_missing = keyword_users[keyword]["missing"]
 372                         if len(kwd_missing):
 373                                 self.printer(keyword, "-", kwd_missing)
 374                                 blankline
 375                 if not self.options['quiet']:
 376                         if self.analyser.mismatched:
 377                                 print("_________________________________________________")
 378                                 print(("The following packages were found to have a \n" +
 379                                         "different recorded ARCH than the current system ARCH"))
 380                                 for cpv in self.analyser.mismatched:
 381                                         print("\t", pp.cpv(cpv))
 382                         print("===================================================")
 383                         print("Total number of keywords in report =",
 384                                 pp.output.red(str(len(keyword_keys))))
 385                         if self.options["verbose"]:
 386                                 print("Total number of installed ebuilds =",
 387                                         pp.output.red(str(len(cpvs))))
 388                         print()
 389
 390
 391         def analyse_packages(self):
 392                 """This will scan the installed packages db and analyse the
 393                 USE flags used for installation and produce a report.
 394
 395                 @type target: string
 396                 @param target: the target to be analysed, one of ["use", "pkguse"]
 397                 """
 398                 system_use = portage.settings["USE"].split()
 399                 if self.options["verbose"]:
 400                         cpvs = VARDB.cpv_all()
 401                         key_width = 45
 402                 else:
 403                         cpvs = get_installed_cpvs()
 404                         key_width = 1
 405
 406                 self.printer = AnalysisPrinter(
 407                                 "packages",
 408                                 self.options["verbose"],
 409                                 key_width=key_width)
 410
 411                 cpvs = sorted(cpvs)
 412                 flags = FlagAnalyzer(
 413                                         system=system_use,
 414                                         filter_defaults=False,
 415                                         target="USE"
 416                                         )
 417
 418                 if self.options["verbose"]:
 419                         print("   cat/pkg-ver                                 USE Flags")
 420                                 #   "app-emulation/emul-linux-x86-sdl-20100915 ...."
 421                         blankline = nl
 422                 elif not self.options['quiet']:
 423                         print("   cat/pkg-ver                                 USE Flags")
 424                         blankline = lambda: None
 425                 for cpv in cpvs:
 426                         (flag_plus, flag_neg, cleaned) = flags.analyse_cpv(cpv)
 427                         if self.options["unset"]:
 428                                 self.printer(cpv, "", (flag_plus, flag_neg, cleaned))
 429                         else:
 430                                 self.printer(cpv, "", (flag_plus, [], cleaned))
 431                 if not self.options['quiet']:
 432                         print("===================================================")
 433                         print("Total number of installed ebuilds =",
 434                                 pp.output.red(str(len([x for x in cpvs]))))
 435                         print()
 436
 437
 438 def main(input_args):
 439         """Common starting method by the analyse master
 440         unless all modules are converted to this class method.
 441
 442         @param input_args: input args as supplied by equery master module.
 443         """
 444         query_module = Analyse()
 445         query_module.run(input_args, gentoolkit.CONFIG['quiet'])
 446
 447 # vim: set ts=4 sw=4 tw=79: