pym/gentoolkit/revdep_rebuild/analyse.py

   1 #!/usr/bin/python
   2
   3 import os
   4 import re
   5 import platform
   6 import glob
   7
   8 from portage.output import bold, red, blue, yellow, green, nocolor
   9
  10 from stuff import scan
  11 from collect import prepare_search_dirs, parse_revdep_config, collect_libraries_from_dir, collect_binaries_from_dir
  12 from assign import assign_packages
  13 from cache import save_cache
  14
  15
  16 def prepare_checks(files_to_check, libraries, bits, cmd_max_args):
  17         ''' Calls scanelf for all files_to_check, then returns found libraries and dependencies
  18         '''
  19
  20         libs = [] # libs found by scanelf
  21         dependencies = [] # list of lists of files (from file_to_check) that uses
  22                                           # library (for dependencies[id] and libs[id] => id==id)
  23
  24         for line in scan(['-M', str(bits), '-nBF', '%F %n'], files_to_check, cmd_max_args):
  25         #call_program(['scanelf', '-M', str(bits), '-nBF', '%F %n',]+files_to_check).strip().split('\n'):
  26                 r = line.strip().split(' ')
  27                 if len(r) < 2: # no dependencies?
  28                         continue
  29
  30                 deps = r[1].split(',')
  31                 for d in deps:
  32                         if d in libs:
  33                                 i = libs.index(d)
  34                                 dependencies[i].append(r[0])
  35                         else:
  36                                 libs.append(d)
  37                                 dependencies.append([r[0],])
  38
  39         return (libs, dependencies)
  40
  41
  42 def extract_dependencies_from_la(la, libraries, to_check, logger):
  43         broken = []
  44
  45         libnames = []
  46         for l in libraries:
  47                 m = re.match('.+\/(.+)\.(so|la|a)(\..+)?', l)
  48                 if m is not None:
  49                         ln = m.group(1)
  50                         if ln not in libnames:
  51                                 libnames += [ln, ]
  52
  53         for f in la:
  54                 if not os.path.exists(f):
  55                         continue
  56
  57                 for line in open(f, 'r').readlines():
  58                         line = line.strip()
  59                         if line.startswith('dependency_libs='):
  60                                 m = re.match("dependency_libs='([^']+)'", line)
  61                                 if m is not None:
  62                                         for el in m.group(1).split(' '):
  63                                                 el = el.strip()
  64                                                 if len(el) < 1 or el.startswith('-L') or el.startswith('-R'):
  65                                                         continue
  66
  67                                                 if el.startswith('-l') and 'lib'+el[2:] in libnames:
  68                                                         pass
  69                                                 elif el in la or el in libraries:
  70                                                         pass
  71                                                 else:
  72                                                         if to_check:
  73                                                                 _break = False
  74                                                                 for tc in to_check:
  75                                                                         if tc in el:
  76                                                                                 _break = True
  77                                                                                 break
  78                                                                 if not _break:
  79                                                                         continue
  80
  81                                                         logger.info(yellow(' * ') + f + ' is broken (requires: ' + bold(el)+')')
  82                                                         broken.append(f)
  83         return broken
  84
  85
  86 def find_broken(found_libs, system_libraries, to_check):
  87         ''' Search for broken libraries.
  88                 Check if system_libraries contains found_libs, where
  89                 system_libraries is list of obsolute pathes and found_libs
  90                 is list of library names.
  91         '''
  92
  93         # join libraries and looking at it as string is way too faster than for-jumping
  94
  95         broken = []
  96         sl = '|'.join(system_libraries)
  97
  98         if not to_check:
  99                 for f in found_libs:
 100                         if f+'|' not in sl:
 101                                 broken.append(found_libs.index(f))
 102         else:
 103                 for tc in to_check:
 104                         for f in found_libs:
 105                                 if tc in f:# and f+'|' not in sl:
 106                                         broken.append(found_libs.index(f))
 107
 108         return broken
 109
 110
 111 def main_checks(found_libs, broken, dependencies, logger):
 112         ''' Checks for broken dependencies.
 113                 found_libs have to be the same as returned by prepare_checks
 114                 broken is list of libraries found by scanelf
 115                 dependencies is the value returned by prepare_checks
 116         '''
 117
 118         broken_pathes = []
 119
 120         for b in broken:
 121                 f = found_libs[b]
 122                 logger.info('Broken files that requires: ' + bold(f))
 123                 for d in dependencies[b]:
 124                         logger.info(yellow(' * ') + d)
 125                         broken_pathes.append(d)
 126         return broken_pathes
 127
 128
 129 def analyse(settings, logger, libraries=None, la_libraries=None,
 130                 libraries_links=None, binaries=None, _libs_to_check=set()):
 131         """Main program body.  It will collect all info and determine the
 132         pkgs needing rebuilding.
 133
 134         @param logger: logger used for logging messages, instance of logging.Logger
 135                                    class. Can be logging (RootLogger).
 136         @param _libs_to_check Libraries that need to be checked only
 137         @rtype list: list of pkgs that need rebuilding
 138         """
 139
 140         if libraries and la_libraries and libraries_links and binaries:
 141                 logger.info(blue(' * ') + bold('Found a valid cache, skipping collecting phase'))
 142         else:
 143                 #TODO: add partial cache (for ex. only libraries) when found for some reason
 144
 145                 logger.warn(green(' * ') + bold('Collecting system binaries and libraries'))
 146                 bin_dirs, lib_dirs = prepare_search_dirs(logger, settings)
 147
 148                 masked_dirs, masked_files, ld = parse_revdep_config(settings['REVDEP_CONFDIR'])
 149                 lib_dirs.update(ld)
 150                 bin_dirs.update(ld)
 151                 masked_dirs = masked_dirs.union(set(['/lib/modules', '/lib32/modules', '/lib64/modules',]))
 152
 153                 logger.info(green(' * ') + bold('Collecting dynamic linking informations'))
 154                 libraries, la_libraries, libraries_links, symlink_pairs = collect_libraries_from_dir(lib_dirs, masked_dirs, logger)
 155                 binaries = collect_binaries_from_dir(bin_dirs, masked_dirs, logger)
 156
 157                 if settings['USE_TMP_FILES']:
 158                         save_cache(logger=logger,
 159                                 to_save={'libraries':libraries, 'la_libraries':la_libraries,
 160                                         'libraries_links':libraries_links, 'binaries':binaries
 161                                 },
 162                         temp_path=settings['DEFAULT_TMP_DIR']
 163                         )
 164
 165
 166         logger.debug('Found '+ str(len(libraries)) + ' libraries (+' + str(len(libraries_links)) + ' symlinks) and ' + str(len(binaries)) + ' binaries')
 167
 168         logger.warn(green(' * ') + bold('Checking dynamic linking consistency'))
 169         logger.debug('Search for ' + str(len(binaries)+len(libraries)) + ' within ' + str(len(libraries)+len(libraries_links)))
 170         libs_and_bins = libraries+binaries
 171
 172         #l = []
 173         #for line in call_program(['scanelf', '-M', '64', '-BF', '%F',] + libraries).strip().split('\n'):
 174                 #l.append(line)
 175         #libraries = l
 176
 177         found_libs = []
 178         dependencies = []
 179
 180         if _libs_to_check:
 181                 nltc = []
 182                 for ltc in _libs_to_check:
 183                         if os.path.isfile(ltc):
 184                                 ltc = scan(['-nBSF', '%S'], [ltc,], settings['CMD_MAX_ARGS'])[0].split()[0]
 185                         nltc += [ltc,]
 186                 _libs_to_check = nltc
 187
 188         _bits, linkg = platform.architecture()
 189         if _bits.startswith('32'):
 190                 bits = 32
 191         elif _bits.startswith('64'):
 192                 bits = 64
 193
 194         import time
 195         broken = []
 196         for av_bits in glob.glob('/lib[0-9]*') or ('/lib32',):
 197                 bits = int(av_bits[4:])
 198
 199                 #_libraries = scan(['-M', str(bits), '-BF', '%F'], libraries+libraries_links, settings['CMD_MAX_ARGS'])
 200                 _libraries = libraries+libraries_links
 201
 202                 found_libs, dependencies = prepare_checks(libs_and_bins, _libraries, bits, settings['CMD_MAX_ARGS'])
 203                 broken = find_broken(found_libs, _libraries, _libs_to_check)
 204
 205                 bits /= 2
 206                 bits = int(bits)
 207
 208         broken_la = extract_dependencies_from_la(la_libraries, libraries+libraries_links, _libs_to_check, logger)
 209
 210
 211         broken_pathes = main_checks(found_libs, broken, dependencies, logger)
 212         broken_pathes += broken_la
 213
 214         logger.warn(green(' * ') + bold('Assign files to packages'))
 215
 216         return assign_packages(broken_pathes, logger, settings)
 217
 218
 219
 220 if __name__ == '__main__':
 221         print("This script shouldn't be called directly")