libbe/storage/util/upgrade.py

   1 # Copyright (C) 2009-2012 Chris Ball <cjb@laptop.org>
   2 #                         Gianluca Montecchi <gian@grys.it>
   3 #                         W. Trevor King <wking@drexel.edu>
   4 #
   5 # This file is part of Bugs Everywhere.
   6 #
   7 # Bugs Everywhere is free software: you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by the Free
   9 # Software Foundation, either version 2 of the License, or (at your option) any
  10 # later version.
  11 #
  12 # Bugs Everywhere is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15 # more details.
  16 #
  17 # You should have received a copy of the GNU General Public License along with
  18 # Bugs Everywhere.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 """
  21 Handle conversion between the various BE storage formats.
  22 """
  23
  24 import codecs
  25 import json
  26 import os, os.path
  27 import sys
  28 import types
  29
  30 import yaml
  31
  32 import libbe
  33 import libbe.bug
  34 import libbe.storage.util.mapfile as mapfile
  35 from libbe.storage import STORAGE_VERSIONS, STORAGE_VERSION
  36 #import libbe.storage.vcs # delay import to avoid cyclic dependency
  37 import libbe.ui.util.editor
  38 import libbe.util
  39 import libbe.util.encoding as encoding
  40 import libbe.util.id
  41
  42
  43 def generate_yaml_mapfile(map):
  44     """From v1.1 to v1.5, BE dirs used YAML mapfiles
  45
  46     >>> generate_yaml_mapfile({'q':'p'})
  47     'q: p\\n\\n'
  48     >>> generate_yaml_mapfile({'q':u'Fran\u00e7ais'})
  49     'q: Fran\\xc3\\xa7ais\\n\\n'
  50     >>> generate_yaml_mapfile({'q':u'hello'})
  51     'q: hello\\n\\n'
  52     """
  53     keys = map.keys()
  54     keys.sort()
  55     for key in keys:
  56         try:
  57             assert not key.startswith('>')
  58             assert('\n' not in key)
  59             assert('=' not in key)
  60             assert(':' not in key)
  61             assert(len(key) > 0)
  62         except AssertionError:
  63             raise ValueError(unicode(key).encode('unicode_escape'))
  64         if '\n' in map[key]:
  65             raise ValueError(unicode(map[key]).encode('unicode_escape'))
  66
  67     lines = []
  68     for key in keys:
  69         lines.append(yaml.safe_dump({key: map[key]},
  70                                     default_flow_style=False,
  71                                     allow_unicode=True))
  72         lines.append('')
  73     return '\n'.join(lines)
  74
  75
  76 def parse_yaml_mapfile(contents):
  77     """From v1.1 to v1.5, BE dirs used YAML mapfiles
  78
  79     >>> parse_yaml_mapfile('q: p\\n\\n')['q']
  80     'p'
  81     >>> parse_yaml_mapfile('q: \\'p\\'\\n\\n')['q']
  82     'p'
  83     >>> contents = generate_yaml_mapfile({'a':'b', 'c':'d', 'e':'f'})
  84     >>> dict = parse_yaml_mapfile(contents)
  85     >>> dict['a']
  86     'b'
  87     >>> dict['c']
  88     'd'
  89     >>> dict['e']
  90     'f'
  91     >>> contents = generate_yaml_mapfile({'q':u'Fran\u00e7ais'})
  92     >>> dict = parse_yaml_mapfile(contents)
  93     >>> dict['q']
  94     u'Fran\\xe7ais'
  95     """
  96     c = yaml.safe_load(contents)
  97     if type(c) == types.StringType:
  98         raise mapfile.InvalidMapfileContents(
  99             'Unable to parse YAML (BE format missmatch?):\n\n%s' % contents)
 100     return c or {}
 101
 102
 103 class Upgrader (object):
 104     "Class for converting between different on-disk BE storage formats."
 105     initial_version = None
 106     final_version = None
 107     def __init__(self, repo):
 108         import libbe.storage.vcs
 109
 110         self.repo = repo
 111         vcs_name = self._get_vcs_name()
 112         if vcs_name == None:
 113             vcs_name = 'None'
 114         self.vcs = libbe.storage.vcs.vcs_by_name(vcs_name)
 115         self.vcs.repo = self.repo
 116         self.vcs.root()
 117
 118     def get_path(self, *args):
 119         """
 120         Return the absolute path using args relative to .be.
 121         """
 122         dir = os.path.join(self.repo, '.be')
 123         if len(args) == 0:
 124             return dir
 125         return os.path.join(dir, *args)
 126
 127     def _get_vcs_name(self):
 128         return None
 129
 130     def check_initial_version(self):
 131         path = self.get_path('version')
 132         version = encoding.get_file_contents(path, decode=True).rstrip('\n')
 133         assert version == self.initial_version, '%s: %s' % (path, version)
 134
 135     def set_version(self):
 136         path = self.get_path('version')
 137         encoding.set_file_contents(path, self.final_version+'\n')
 138         self.vcs._vcs_update(path)
 139
 140     def upgrade(self):
 141         print >> sys.stderr, 'upgrading bugdir from "%s" to "%s"' \
 142             % (self.initial_version, self.final_version)
 143         self.check_initial_version()
 144         self.set_version()
 145         self._upgrade()
 146
 147     def _upgrade(self):
 148         raise NotImplementedError
 149
 150
 151 class Upgrade_1_0_to_1_1 (Upgrader):
 152     initial_version = "Bugs Everywhere Tree 1 0"
 153     final_version = "Bugs Everywhere Directory v1.1"
 154     def _get_vcs_name(self):
 155         path = self.get_path('settings')
 156         settings = encoding.get_file_contents(path)
 157         for line in settings.splitlines(False):
 158             fields = line.split('=')
 159             if len(fields) == 2 and fields[0] == 'rcs_name':
 160                 return fields[1]
 161         return None
 162
 163     def _upgrade_mapfile(self, path):
 164         contents = encoding.get_file_contents(path, decode=True)
 165         old_format = False
 166         for line in contents.splitlines():
 167             if len(line.split('=')) == 2:
 168                 old_format = True
 169                 break
 170         if old_format == True:
 171             # translate to YAML.
 172             newlines = []
 173             for line in contents.splitlines():
 174                 line = line.rstrip('\n')
 175                 if len(line) == 0:
 176                     continue
 177                 fields = line.split("=")
 178                 if len(fields) == 2:
 179                     key,value = fields
 180                     newlines.append('%s: "%s"' % (key, value.replace('"','\\"')))
 181                 else:
 182                     newlines.append(line)
 183             contents = '\n'.join(newlines)
 184             # load the YAML and save
 185             map = parse_yaml_mapfile(contents)
 186             if type(map) == types.StringType:
 187                 raise ValueError((path, contents))
 188             contents = generate_yaml_mapfile(map)
 189             encoding.set_file_contents(path, contents)
 190             self.vcs._vcs_update(path)
 191
 192     def _upgrade(self):
 193         """
 194         Comment value field "From" -> "Author".
 195         Homegrown mapfile -> YAML.
 196         """
 197         path = self.get_path('settings')
 198         self._upgrade_mapfile(path)
 199         for bug_uuid in os.listdir(self.get_path('bugs')):
 200             path = self.get_path('bugs', bug_uuid, 'values')
 201             self._upgrade_mapfile(path)
 202             c_path = ['bugs', bug_uuid, 'comments']
 203             if not os.path.exists(self.get_path(*c_path)):
 204                 continue # no comments for this bug
 205             for comment_uuid in os.listdir(self.get_path(*c_path)):
 206                 path_list = c_path + [comment_uuid, 'values']
 207                 path = self.get_path(*path_list)
 208                 self._upgrade_mapfile(path)
 209                 settings = mapfile.parse(
 210                     encoding.get_file_contents(path))
 211                 if 'From' in settings:
 212                     settings['Author'] = settings.pop('From')
 213                     encoding.set_file_contents(
 214                         path, generate_yaml_mapfile(settings))
 215                     self.vcs._vcs_update(path)
 216
 217
 218 class Upgrade_1_1_to_1_2 (Upgrader):
 219     initial_version = "Bugs Everywhere Directory v1.1"
 220     final_version = "Bugs Everywhere Directory v1.2"
 221     def _get_vcs_name(self):
 222         path = self.get_path('settings')
 223         settings = parse_yaml_mapfile(encoding.get_file_contents(path))
 224         if 'rcs_name' in settings:
 225             return settings['rcs_name']
 226         return None
 227
 228     def _upgrade(self):
 229         """
 230         BugDir settings field "rcs_name" -> "vcs_name".
 231         """
 232         path = self.get_path('settings')
 233         settings = parse_yaml_mapfile(encoding.get_file_contents(path))
 234         if 'rcs_name' in settings:
 235             settings['vcs_name'] = settings.pop('rcs_name')
 236             encoding.set_file_contents(path, generate_yaml_mapfile(settings))
 237             self.vcs._vcs_update(path)
 238
 239 class Upgrade_1_2_to_1_3 (Upgrader):
 240     initial_version = "Bugs Everywhere Directory v1.2"
 241     final_version = "Bugs Everywhere Directory v1.3"
 242     def __init__(self, *args, **kwargs):
 243         Upgrader.__init__(self, *args, **kwargs)
 244         self._targets = {} # key: target text,value: new target bug
 245
 246     def _get_vcs_name(self):
 247         path = self.get_path('settings')
 248         settings = parse_yaml_mapfile(encoding.get_file_contents(path))
 249         if 'vcs_name' in settings:
 250             return settings['vcs_name']
 251         return None
 252
 253     def _save_bug_settings(self, bug):
 254         # The target bugs don't have comments
 255         path = self.get_path('bugs', bug.uuid, 'values')
 256         if not os.path.exists(path):
 257             self.vcs._add_path(path, directory=False)
 258         path = self.get_path('bugs', bug.uuid, 'values')
 259         mf = generate_yaml_mapfile(bug._get_saved_settings())
 260         encoding.set_file_contents(path, mf)
 261         self.vcs._vcs_update(path)
 262
 263     def _target_bug(self, target_text):
 264         if target_text not in self._targets:
 265             bug = libbe.bug.Bug(summary=target_text)
 266             bug.severity = 'target'
 267             self._targets[target_text] = bug
 268         return self._targets[target_text]
 269
 270     def _upgrade_bugdir_mapfile(self):
 271         path = self.get_path('settings')
 272         mf = encoding.get_file_contents(path)
 273         if mf == libbe.util.InvalidObject:
 274             return # settings file does not exist
 275         settings = parse_yaml_mapfile(mf)
 276         if 'target' in settings:
 277             settings['target'] = self._target_bug(settings['target']).uuid
 278             mf = generate_yaml_mapfile(settings)
 279             encoding.set_file_contents(path, mf)
 280             self.vcs._vcs_update(path)
 281
 282     def _upgrade_bug_mapfile(self, bug_uuid):
 283         import libbe.command.depend as dep
 284         path = self.get_path('bugs', bug_uuid, 'values')
 285         mf = encoding.get_file_contents(path)
 286         if mf == libbe.util.InvalidObject:
 287             return # settings file does not exist
 288         settings = parse_yaml_mapfile(mf)
 289         if 'target' in settings:
 290             target_bug = self._target_bug(settings['target'])
 291
 292             blocked_by_string = '%s%s' % (dep.BLOCKED_BY_TAG, bug_uuid)
 293             dep._add_remove_extra_string(target_bug, blocked_by_string, add=True)
 294             blocks_string = dep._generate_blocks_string(target_bug)
 295             estrs = settings.get('extra_strings', [])
 296             estrs.append(blocks_string)
 297             settings['extra_strings'] = sorted(estrs)
 298
 299             settings.pop('target')
 300             mf = generate_yaml_mapfile(settings)
 301             encoding.set_file_contents(path, mf)
 302             self.vcs._vcs_update(path)
 303
 304     def _upgrade(self):
 305         """
 306         Bug value field "target" -> target bugs.
 307         Bugdir value field "target" -> pointer to current target bug.
 308         """
 309         for bug_uuid in os.listdir(self.get_path('bugs')):
 310             self._upgrade_bug_mapfile(bug_uuid)
 311         self._upgrade_bugdir_mapfile()
 312         for bug in self._targets.values():
 313             self._save_bug_settings(bug)
 314
 315 class Upgrade_1_3_to_1_4 (Upgrader):
 316     initial_version = "Bugs Everywhere Directory v1.3"
 317     final_version = "Bugs Everywhere Directory v1.4"
 318     def _get_vcs_name(self):
 319         path = self.get_path('settings')
 320         settings = parse_yaml_mapfile(encoding.get_file_contents(path))
 321         if 'vcs_name' in settings:
 322             return settings['vcs_name']
 323         return None
 324
 325     def _upgrade(self):
 326         """
 327         add new directory "./be/BUGDIR-UUID"
 328         "./be/bugs" -> "./be/BUGDIR-UUID/bugs"
 329         "./be/settings" -> "./be/BUGDIR-UUID/settings"
 330         """
 331         self.repo = os.path.abspath(self.repo)
 332         basenames = [p for p in os.listdir(self.get_path())]
 333         if not 'bugs' in basenames and not 'settings' in basenames \
 334                 and len([p for p in basenames if len(p)==36]) == 1:
 335             return # the user has upgraded the directory.
 336         basenames = [p for p in basenames if p in ['bugs','settings']]
 337         uuid = libbe.util.id.uuid_gen()
 338         add = [self.get_path(uuid)]
 339         move = [(self.get_path(p), self.get_path(uuid, p)) for p in basenames]
 340         msg = ['Upgrading BE directory version v1.3 to v1.4',
 341                '',
 342                "Because BE's VCS drivers don't support 'move',",
 343                'please make the following changes with your VCS',
 344                'and re-run BE.  Note that you can choose a different',
 345                'bugdir UUID to preserve uniformity across branches',
 346                'of a distributed repository.'
 347                '',
 348                'add',
 349                '  ' + '\n  '.join(add),
 350                'move',
 351                '  ' + '\n  '.join(['%s %s' % (a,b) for a,b in move]),
 352                ]
 353         self.vcs._cached_path_id.destroy()
 354         raise Exception('Need user assistance\n%s' % '\n'.join(msg))
 355
 356
 357 class Upgrade_1_4_to_1_5 (Upgrader):
 358     initial_version = "Bugs Everywhere Directory v1.4"
 359     final_version = "Bugs Everywhere Directory v1.5"
 360     def _get_vcs_name(self):
 361         path = self.get_path('settings')
 362         for p in os.listdir(self.get_path()):  # check each bugdir's settings
 363             path = os.path.join(self.get_path(), p)
 364             if os.path.isdir(path):
 365                 settings_path = os.path.join(path, 'settings')
 366                 if os.path.isfile(settings_path):
 367                     settings = parse_yaml_mapfile(encoding.get_file_contents(
 368                             settings_path))
 369                     if 'vcs_name' in settings:
 370                         return settings['vcs_name']  # first entry we found
 371         return None
 372
 373     def _upgrade(self):
 374         """
 375         convert YAML settings to JSON (much faster parsing)
 376         "./be/BUGDIR-UUID/settings"
 377         "./be/BUGDIR-UUID/bugs/BUG-UUID/values"
 378         "./be/BUGDIR-UUID/bugs/BUG-UUID/comments/COMMENT-UUID/values"
 379         """
 380         self.repo = os.path.abspath(self.repo)
 381         basenames = [p for p in os.listdir(self.get_path())]
 382         for dirpath,dirnames,filenames in os.walk(self.get_path()):
 383             for filename in filenames:
 384                 if filename in ['settings', 'values']:
 385                     self._upgrade_mapfile(os.path.join(dirpath, filename))
 386
 387     def _upgrade_mapfile(self, path):
 388         contents = encoding.get_file_contents(path)
 389         data = parse_yaml_mapfile(contents)
 390         contents = mapfile.generate(data)
 391         encoding.set_file_contents(path, contents)
 392         self.vcs._vcs_update(path)
 393
 394
 395 upgraders = [Upgrade_1_0_to_1_1,
 396              Upgrade_1_1_to_1_2,
 397              Upgrade_1_2_to_1_3,
 398              Upgrade_1_3_to_1_4,
 399              Upgrade_1_4_to_1_5]
 400
 401 upgrade_classes = {}
 402 for upgrader in upgraders:
 403     upgrade_classes[(upgrader.initial_version,upgrader.final_version)]=upgrader
 404
 405 def upgrade(path, current_version,
 406             target_version=STORAGE_VERSION):
 407     """
 408     Call the appropriate upgrade function to convert current_version
 409     to target_version.  If a direct conversion function does not exist,
 410     use consecutive conversion functions.
 411     """
 412     if current_version not in STORAGE_VERSIONS:
 413         raise NotImplementedError, \
 414             "Cannot handle version '%s' yet." % current_version
 415     if target_version not in STORAGE_VERSIONS:
 416         raise NotImplementedError, \
 417             "Cannot handle version '%s' yet." % current_version
 418
 419     if (current_version, target_version) in upgrade_classes:
 420         # direct conversion
 421         upgrade_class = upgrade_classes[(current_version, target_version)]
 422         u = upgrade_class(path)
 423         u.upgrade()
 424     else:
 425         # consecutive single-step conversion
 426         i = STORAGE_VERSIONS.index(current_version)
 427         while True:
 428             version_a = STORAGE_VERSIONS[i]
 429             version_b = STORAGE_VERSIONS[i+1]
 430             try:
 431                 upgrade_class = upgrade_classes[(version_a, version_b)]
 432             except KeyError:
 433                 raise NotImplementedError, \
 434                     "Cannot convert version '%s' to '%s' yet." \
 435                     % (version_a, version_b)
 436             u = upgrade_class(path)
 437             u.upgrade()
 438             if version_b == target_version:
 439                 break
 440             i += 1