git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re, shutil
  14
  15 verbose = False
  16
  17 # Only labels/tags matching this will be imported/exported
  18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = ["p4"]
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += ["-u",user]
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += ["-P", password]
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += ["-p", port]
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += ["-H", host]
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += ["-c", client]
  48
  49
  50     if isinstance(cmd,basestring):
  51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  52     else:
  53         real_cmd += cmd
  54     return real_cmd
  55
  56 def chdir(dir):
  57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  58     # not using the shell, we have to set it ourselves.  This path could
  59     # be relative, so go there first, then figure out where we ended up.
  60     os.chdir(dir)
  61     os.environ['PWD'] = os.getcwd()
  62
  63 def die(msg):
  64     if verbose:
  65         raise Exception(msg)
  66     else:
  67         sys.stderr.write(msg + "\n")
  68         sys.exit(1)
  69
  70 def write_pipe(c, stdin):
  71     if verbose:
  72         sys.stderr.write('Writing pipe: %s\n' % str(c))
  73
  74     expand = isinstance(c,basestring)
  75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  76     pipe = p.stdin
  77     val = pipe.write(stdin)
  78     pipe.close()
  79     if p.wait():
  80         die('Command failed: %s' % str(c))
  81
  82     return val
  83
  84 def p4_write_pipe(c, stdin):
  85     real_cmd = p4_build_cmd(c)
  86     return write_pipe(real_cmd, stdin)
  87
  88 def read_pipe(c, ignore_error=False):
  89     if verbose:
  90         sys.stderr.write('Reading pipe: %s\n' % str(c))
  91
  92     expand = isinstance(c,basestring)
  93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  94     pipe = p.stdout
  95     val = pipe.read()
  96     if p.wait() and not ignore_error:
  97         die('Command failed: %s' % str(c))
  98
  99     return val
 100
 101 def p4_read_pipe(c, ignore_error=False):
 102     real_cmd = p4_build_cmd(c)
 103     return read_pipe(real_cmd, ignore_error)
 104
 105 def read_pipe_lines(c):
 106     if verbose:
 107         sys.stderr.write('Reading pipe: %s\n' % str(c))
 108
 109     expand = isinstance(c, basestring)
 110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 111     pipe = p.stdout
 112     val = pipe.readlines()
 113     if pipe.close() or p.wait():
 114         die('Command failed: %s' % str(c))
 115
 116     return val
 117
 118 def p4_read_pipe_lines(c):
 119     """Specifically invoke p4 on the command supplied. """
 120     real_cmd = p4_build_cmd(c)
 121     return read_pipe_lines(real_cmd)
 122
 123 def system(cmd):
 124     expand = isinstance(cmd,basestring)
 125     if verbose:
 126         sys.stderr.write("executing %s\n" % str(cmd))
 127     subprocess.check_call(cmd, shell=expand)
 128
 129 def p4_system(cmd):
 130     """Specifically invoke p4 as the system command. """
 131     real_cmd = p4_build_cmd(cmd)
 132     expand = isinstance(real_cmd, basestring)
 133     subprocess.check_call(real_cmd, shell=expand)
 134
 135 def p4_integrate(src, dest):
 136     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 137
 138 def p4_sync(f, *options):
 139     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 140
 141 def p4_add(f):
 142     # forcibly add file names with wildcards
 143     if wildcard_present(f):
 144         p4_system(["add", "-f", f])
 145     else:
 146         p4_system(["add", f])
 147
 148 def p4_delete(f):
 149     p4_system(["delete", wildcard_encode(f)])
 150
 151 def p4_edit(f):
 152     p4_system(["edit", wildcard_encode(f)])
 153
 154 def p4_revert(f):
 155     p4_system(["revert", wildcard_encode(f)])
 156
 157 def p4_reopen(type, f):
 158     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 159
 160 #
 161 # Canonicalize the p4 type and return a tuple of the
 162 # base type, plus any modifiers.  See "p4 help filetypes"
 163 # for a list and explanation.
 164 #
 165 def split_p4_type(p4type):
 166
 167     p4_filetypes_historical = {
 168         "ctempobj": "binary+Sw",
 169         "ctext": "text+C",
 170         "cxtext": "text+Cx",
 171         "ktext": "text+k",
 172         "kxtext": "text+kx",
 173         "ltext": "text+F",
 174         "tempobj": "binary+FSw",
 175         "ubinary": "binary+F",
 176         "uresource": "resource+F",
 177         "uxbinary": "binary+Fx",
 178         "xbinary": "binary+x",
 179         "xltext": "text+Fx",
 180         "xtempobj": "binary+Swx",
 181         "xtext": "text+x",
 182         "xunicode": "unicode+x",
 183         "xutf16": "utf16+x",
 184     }
 185     if p4type in p4_filetypes_historical:
 186         p4type = p4_filetypes_historical[p4type]
 187     mods = ""
 188     s = p4type.split("+")
 189     base = s[0]
 190     mods = ""
 191     if len(s) > 1:
 192         mods = s[1]
 193     return (base, mods)
 194
 195 #
 196 # return the raw p4 type of a file (text, text+ko, etc)
 197 #
 198 def p4_type(file):
 199     results = p4CmdList(["fstat", "-T", "headType", file])
 200     return results[0]['headType']
 201
 202 #
 203 # Given a type base and modifier, return a regexp matching
 204 # the keywords that can be expanded in the file
 205 #
 206 def p4_keywords_regexp_for_type(base, type_mods):
 207     if base in ("text", "unicode", "binary"):
 208         kwords = None
 209         if "ko" in type_mods:
 210             kwords = 'Id|Header'
 211         elif "k" in type_mods:
 212             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 213         else:
 214             return None
 215         pattern = r"""
 216             \$              # Starts with a dollar, followed by...
 217             (%s)            # one of the keywords, followed by...
 218             (:[^$]+)?       # possibly an old expansion, followed by...
 219             \$              # another dollar
 220             """ % kwords
 221         return pattern
 222     else:
 223         return None
 224
 225 #
 226 # Given a file, return a regexp matching the possible
 227 # RCS keywords that will be expanded, or None for files
 228 # with kw expansion turned off.
 229 #
 230 def p4_keywords_regexp_for_file(file):
 231     if not os.path.exists(file):
 232         return None
 233     else:
 234         (type_base, type_mods) = split_p4_type(p4_type(file))
 235         return p4_keywords_regexp_for_type(type_base, type_mods)
 236
 237 def setP4ExecBit(file, mode):
 238     # Reopens an already open file and changes the execute bit to match
 239     # the execute bit setting in the passed in mode.
 240
 241     p4Type = "+x"
 242
 243     if not isModeExec(mode):
 244         p4Type = getP4OpenedType(file)
 245         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 246         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 247         if p4Type[-1] == "+":
 248             p4Type = p4Type[0:-1]
 249
 250     p4_reopen(p4Type, file)
 251
 252 def getP4OpenedType(file):
 253     # Returns the perforce file type for the given file.
 254
 255     result = p4_read_pipe(["opened", wildcard_encode(file)])
 256     match = re.match(".*\((.+)\)\r?$", result)
 257     if match:
 258         return match.group(1)
 259     else:
 260         die("Could not determine file type for %s (result: '%s')" % (file, result))
 261
 262 # Return the set of all p4 labels
 263 def getP4Labels(depotPaths):
 264     labels = set()
 265     if isinstance(depotPaths,basestring):
 266         depotPaths = [depotPaths]
 267
 268     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 269         label = l['label']
 270         labels.add(label)
 271
 272     return labels
 273
 274 # Return the set of all git tags
 275 def getGitTags():
 276     gitTags = set()
 277     for line in read_pipe_lines(["git", "tag"]):
 278         tag = line.strip()
 279         gitTags.add(tag)
 280     return gitTags
 281
 282 def diffTreePattern():
 283     # This is a simple generator for the diff tree regex pattern. This could be
 284     # a class variable if this and parseDiffTreeEntry were a part of a class.
 285     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 286     while True:
 287         yield pattern
 288
 289 def parseDiffTreeEntry(entry):
 290     """Parses a single diff tree entry into its component elements.
 291
 292     See git-diff-tree(1) manpage for details about the format of the diff
 293     output. This method returns a dictionary with the following elements:
 294
 295     src_mode - The mode of the source file
 296     dst_mode - The mode of the destination file
 297     src_sha1 - The sha1 for the source file
 298     dst_sha1 - The sha1 fr the destination file
 299     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 300     status_score - The score for the status (applicable for 'C' and 'R'
 301                    statuses). This is None if there is no score.
 302     src - The path for the source file.
 303     dst - The path for the destination file. This is only present for
 304           copy or renames. If it is not present, this is None.
 305
 306     If the pattern is not matched, None is returned."""
 307
 308     match = diffTreePattern().next().match(entry)
 309     if match:
 310         return {
 311             'src_mode': match.group(1),
 312             'dst_mode': match.group(2),
 313             'src_sha1': match.group(3),
 314             'dst_sha1': match.group(4),
 315             'status': match.group(5),
 316             'status_score': match.group(6),
 317             'src': match.group(7),
 318             'dst': match.group(10)
 319         }
 320     return None
 321
 322 def isModeExec(mode):
 323     # Returns True if the given git mode represents an executable file,
 324     # otherwise False.
 325     return mode[-3:] == "755"
 326
 327 def isModeExecChanged(src_mode, dst_mode):
 328     return isModeExec(src_mode) != isModeExec(dst_mode)
 329
 330 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 331
 332     if isinstance(cmd,basestring):
 333         cmd = "-G " + cmd
 334         expand = True
 335     else:
 336         cmd = ["-G"] + cmd
 337         expand = False
 338
 339     cmd = p4_build_cmd(cmd)
 340     if verbose:
 341         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 342
 343     # Use a temporary file to avoid deadlocks without
 344     # subprocess.communicate(), which would put another copy
 345     # of stdout into memory.
 346     stdin_file = None
 347     if stdin is not None:
 348         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 349         if isinstance(stdin,basestring):
 350             stdin_file.write(stdin)
 351         else:
 352             for i in stdin:
 353                 stdin_file.write(i + '\n')
 354         stdin_file.flush()
 355         stdin_file.seek(0)
 356
 357     p4 = subprocess.Popen(cmd,
 358                           shell=expand,
 359                           stdin=stdin_file,
 360                           stdout=subprocess.PIPE)
 361
 362     result = []
 363     try:
 364         while True:
 365             entry = marshal.load(p4.stdout)
 366             if cb is not None:
 367                 cb(entry)
 368             else:
 369                 result.append(entry)
 370     except EOFError:
 371         pass
 372     exitCode = p4.wait()
 373     if exitCode != 0:
 374         entry = {}
 375         entry["p4ExitCode"] = exitCode
 376         result.append(entry)
 377
 378     return result
 379
 380 def p4Cmd(cmd):
 381     list = p4CmdList(cmd)
 382     result = {}
 383     for entry in list:
 384         result.update(entry)
 385     return result;
 386
 387 def p4Where(depotPath):
 388     if not depotPath.endswith("/"):
 389         depotPath += "/"
 390     depotPath = depotPath + "..."
 391     outputList = p4CmdList(["where", depotPath])
 392     output = None
 393     for entry in outputList:
 394         if "depotFile" in entry:
 395             if entry["depotFile"] == depotPath:
 396                 output = entry
 397                 break
 398         elif "data" in entry:
 399             data = entry.get("data")
 400             space = data.find(" ")
 401             if data[:space] == depotPath:
 402                 output = entry
 403                 break
 404     if output == None:
 405         return ""
 406     if output["code"] == "error":
 407         return ""
 408     clientPath = ""
 409     if "path" in output:
 410         clientPath = output.get("path")
 411     elif "data" in output:
 412         data = output.get("data")
 413         lastSpace = data.rfind(" ")
 414         clientPath = data[lastSpace + 1:]
 415
 416     if clientPath.endswith("..."):
 417         clientPath = clientPath[:-3]
 418     return clientPath
 419
 420 def currentGitBranch():
 421     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 422
 423 def isValidGitDir(path):
 424     if (os.path.exists(path + "/HEAD")
 425         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 426         return True;
 427     return False
 428
 429 def parseRevision(ref):
 430     return read_pipe("git rev-parse %s" % ref).strip()
 431
 432 def branchExists(ref):
 433     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 434                      ignore_error=True)
 435     return len(rev) > 0
 436
 437 def extractLogMessageFromGitCommit(commit):
 438     logMessage = ""
 439
 440     ## fixme: title is first line of commit, not 1st paragraph.
 441     foundTitle = False
 442     for log in read_pipe_lines("git cat-file commit %s" % commit):
 443        if not foundTitle:
 444            if len(log) == 1:
 445                foundTitle = True
 446            continue
 447
 448        logMessage += log
 449     return logMessage
 450
 451 def extractSettingsGitLog(log):
 452     values = {}
 453     for line in log.split("\n"):
 454         line = line.strip()
 455         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 456         if not m:
 457             continue
 458
 459         assignments = m.group(1).split (':')
 460         for a in assignments:
 461             vals = a.split ('=')
 462             key = vals[0].strip()
 463             val = ('='.join (vals[1:])).strip()
 464             if val.endswith ('\"') and val.startswith('"'):
 465                 val = val[1:-1]
 466
 467             values[key] = val
 468
 469     paths = values.get("depot-paths")
 470     if not paths:
 471         paths = values.get("depot-path")
 472     if paths:
 473         values['depot-paths'] = paths.split(',')
 474     return values
 475
 476 def gitBranchExists(branch):
 477     proc = subprocess.Popen(["git", "rev-parse", branch],
 478                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 479     return proc.wait() == 0;
 480
 481 _gitConfig = {}
 482 def gitConfig(key, args = None): # set args to "--bool", for instance
 483     if not _gitConfig.has_key(key):
 484         argsFilter = ""
 485         if args != None:
 486             argsFilter = "%s " % args
 487         cmd = "git config %s%s" % (argsFilter, key)
 488         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 489     return _gitConfig[key]
 490
 491 def gitConfigList(key):
 492     if not _gitConfig.has_key(key):
 493         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 494     return _gitConfig[key]
 495
 496 def p4BranchesInGit(branchesAreInRemotes = True):
 497     branches = {}
 498
 499     cmdline = "git rev-parse --symbolic "
 500     if branchesAreInRemotes:
 501         cmdline += " --remotes"
 502     else:
 503         cmdline += " --branches"
 504
 505     for line in read_pipe_lines(cmdline):
 506         line = line.strip()
 507
 508         ## only import to p4/
 509         if not line.startswith('p4/') or line == "p4/HEAD":
 510             continue
 511         branch = line
 512
 513         # strip off p4
 514         branch = re.sub ("^p4/", "", line)
 515
 516         branches[branch] = parseRevision(line)
 517     return branches
 518
 519 def findUpstreamBranchPoint(head = "HEAD"):
 520     branches = p4BranchesInGit()
 521     # map from depot-path to branch name
 522     branchByDepotPath = {}
 523     for branch in branches.keys():
 524         tip = branches[branch]
 525         log = extractLogMessageFromGitCommit(tip)
 526         settings = extractSettingsGitLog(log)
 527         if settings.has_key("depot-paths"):
 528             paths = ",".join(settings["depot-paths"])
 529             branchByDepotPath[paths] = "remotes/p4/" + branch
 530
 531     settings = None
 532     parent = 0
 533     while parent < 65535:
 534         commit = head + "~%s" % parent
 535         log = extractLogMessageFromGitCommit(commit)
 536         settings = extractSettingsGitLog(log)
 537         if settings.has_key("depot-paths"):
 538             paths = ",".join(settings["depot-paths"])
 539             if branchByDepotPath.has_key(paths):
 540                 return [branchByDepotPath[paths], settings]
 541
 542         parent = parent + 1
 543
 544     return ["", settings]
 545
 546 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 547     if not silent:
 548         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 549                % localRefPrefix)
 550
 551     originPrefix = "origin/p4/"
 552
 553     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 554         line = line.strip()
 555         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 556             continue
 557
 558         headName = line[len(originPrefix):]
 559         remoteHead = localRefPrefix + headName
 560         originHead = line
 561
 562         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 563         if (not original.has_key('depot-paths')
 564             or not original.has_key('change')):
 565             continue
 566
 567         update = False
 568         if not gitBranchExists(remoteHead):
 569             if verbose:
 570                 print "creating %s" % remoteHead
 571             update = True
 572         else:
 573             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 574             if settings.has_key('change') > 0:
 575                 if settings['depot-paths'] == original['depot-paths']:
 576                     originP4Change = int(original['change'])
 577                     p4Change = int(settings['change'])
 578                     if originP4Change > p4Change:
 579                         print ("%s (%s) is newer than %s (%s). "
 580                                "Updating p4 branch from origin."
 581                                % (originHead, originP4Change,
 582                                   remoteHead, p4Change))
 583                         update = True
 584                 else:
 585                     print ("Ignoring: %s was imported from %s while "
 586                            "%s was imported from %s"
 587                            % (originHead, ','.join(original['depot-paths']),
 588                               remoteHead, ','.join(settings['depot-paths'])))
 589
 590         if update:
 591             system("git update-ref %s %s" % (remoteHead, originHead))
 592
 593 def originP4BranchesExist():
 594         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 595
 596 def p4ChangesForPaths(depotPaths, changeRange):
 597     assert depotPaths
 598     cmd = ['changes']
 599     for p in depotPaths:
 600         cmd += ["%s...%s" % (p, changeRange)]
 601     output = p4_read_pipe_lines(cmd)
 602
 603     changes = {}
 604     for line in output:
 605         changeNum = int(line.split(" ")[1])
 606         changes[changeNum] = True
 607
 608     changelist = changes.keys()
 609     changelist.sort()
 610     return changelist
 611
 612 def p4PathStartsWith(path, prefix):
 613     # This method tries to remedy a potential mixed-case issue:
 614     #
 615     # If UserA adds  //depot/DirA/file1
 616     # and UserB adds //depot/dira/file2
 617     #
 618     # we may or may not have a problem. If you have core.ignorecase=true,
 619     # we treat DirA and dira as the same directory
 620     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 621     if ignorecase:
 622         return path.lower().startswith(prefix.lower())
 623     return path.startswith(prefix)
 624
 625 def getClientSpec():
 626     """Look at the p4 client spec, create a View() object that contains
 627        all the mappings, and return it."""
 628
 629     specList = p4CmdList("client -o")
 630     if len(specList) != 1:
 631         die('Output from "client -o" is %d lines, expecting 1' %
 632             len(specList))
 633
 634     # dictionary of all client parameters
 635     entry = specList[0]
 636
 637     # just the keys that start with "View"
 638     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 639
 640     # hold this new View
 641     view = View()
 642
 643     # append the lines, in order, to the view
 644     for view_num in range(len(view_keys)):
 645         k = "View%d" % view_num
 646         if k not in view_keys:
 647             die("Expected view key %s missing" % k)
 648         view.append(entry[k])
 649
 650     return view
 651
 652 def getClientRoot():
 653     """Grab the client directory."""
 654
 655     output = p4CmdList("client -o")
 656     if len(output) != 1:
 657         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 658
 659     entry = output[0]
 660     if "Root" not in entry:
 661         die('Client has no "Root"')
 662
 663     return entry["Root"]
 664
 665 #
 666 # P4 wildcards are not allowed in filenames.  P4 complains
 667 # if you simply add them, but you can force it with "-f", in
 668 # which case it translates them into %xx encoding internally.
 669 #
 670 def wildcard_decode(path):
 671     # Search for and fix just these four characters.  Do % last so
 672     # that fixing it does not inadvertently create new %-escapes.
 673     # Cannot have * in a filename in windows; untested as to
 674     # what p4 would do in such a case.
 675     if not platform.system() == "Windows":
 676         path = path.replace("%2A", "*")
 677     path = path.replace("%23", "#") \
 678                .replace("%40", "@") \
 679                .replace("%25", "%")
 680     return path
 681
 682 def wildcard_encode(path):
 683     # do % first to avoid double-encoding the %s introduced here
 684     path = path.replace("%", "%25") \
 685                .replace("*", "%2A") \
 686                .replace("#", "%23") \
 687                .replace("@", "%40")
 688     return path
 689
 690 def wildcard_present(path):
 691     return path.translate(None, "*#@%") != path
 692
 693 class Command:
 694     def __init__(self):
 695         self.usage = "usage: %prog [options]"
 696         self.needsGit = True
 697         self.verbose = False
 698
 699 class P4UserMap:
 700     def __init__(self):
 701         self.userMapFromPerforceServer = False
 702         self.myP4UserId = None
 703
 704     def p4UserId(self):
 705         if self.myP4UserId:
 706             return self.myP4UserId
 707
 708         results = p4CmdList("user -o")
 709         for r in results:
 710             if r.has_key('User'):
 711                 self.myP4UserId = r['User']
 712                 return r['User']
 713         die("Could not find your p4 user id")
 714
 715     def p4UserIsMe(self, p4User):
 716         # return True if the given p4 user is actually me
 717         me = self.p4UserId()
 718         if not p4User or p4User != me:
 719             return False
 720         else:
 721             return True
 722
 723     def getUserCacheFilename(self):
 724         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 725         return home + "/.gitp4-usercache.txt"
 726
 727     def getUserMapFromPerforceServer(self):
 728         if self.userMapFromPerforceServer:
 729             return
 730         self.users = {}
 731         self.emails = {}
 732
 733         for output in p4CmdList("users"):
 734             if not output.has_key("User"):
 735                 continue
 736             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 737             self.emails[output["Email"]] = output["User"]
 738
 739
 740         s = ''
 741         for (key, val) in self.users.items():
 742             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 743
 744         open(self.getUserCacheFilename(), "wb").write(s)
 745         self.userMapFromPerforceServer = True
 746
 747     def loadUserMapFromCache(self):
 748         self.users = {}
 749         self.userMapFromPerforceServer = False
 750         try:
 751             cache = open(self.getUserCacheFilename(), "rb")
 752             lines = cache.readlines()
 753             cache.close()
 754             for line in lines:
 755                 entry = line.strip().split("\t")
 756                 self.users[entry[0]] = entry[1]
 757         except IOError:
 758             self.getUserMapFromPerforceServer()
 759
 760 class P4Debug(Command):
 761     def __init__(self):
 762         Command.__init__(self)
 763         self.options = []
 764         self.description = "A tool to debug the output of p4 -G."
 765         self.needsGit = False
 766
 767     def run(self, args):
 768         j = 0
 769         for output in p4CmdList(args):
 770             print 'Element: %d' % j
 771             j += 1
 772             print output
 773         return True
 774
 775 class P4RollBack(Command):
 776     def __init__(self):
 777         Command.__init__(self)
 778         self.options = [
 779             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 780         ]
 781         self.description = "A tool to debug the multi-branch import. Don't use :)"
 782         self.rollbackLocalBranches = False
 783
 784     def run(self, args):
 785         if len(args) != 1:
 786             return False
 787         maxChange = int(args[0])
 788
 789         if "p4ExitCode" in p4Cmd("changes -m 1"):
 790             die("Problems executing p4");
 791
 792         if self.rollbackLocalBranches:
 793             refPrefix = "refs/heads/"
 794             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 795         else:
 796             refPrefix = "refs/remotes/"
 797             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 798
 799         for line in lines:
 800             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 801                 line = line.strip()
 802                 ref = refPrefix + line
 803                 log = extractLogMessageFromGitCommit(ref)
 804                 settings = extractSettingsGitLog(log)
 805
 806                 depotPaths = settings['depot-paths']
 807                 change = settings['change']
 808
 809                 changed = False
 810
 811                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 812                                                            for p in depotPaths]))) == 0:
 813                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 814                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 815                     continue
 816
 817                 while change and int(change) > maxChange:
 818                     changed = True
 819                     if self.verbose:
 820                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 821                     system("git update-ref %s \"%s^\"" % (ref, ref))
 822                     log = extractLogMessageFromGitCommit(ref)
 823                     settings =  extractSettingsGitLog(log)
 824
 825
 826                     depotPaths = settings['depot-paths']
 827                     change = settings['change']
 828
 829                 if changed:
 830                     print "%s rewound to %s" % (ref, change)
 831
 832         return True
 833
 834 class P4Submit(Command, P4UserMap):
 835     def __init__(self):
 836         Command.__init__(self)
 837         P4UserMap.__init__(self)
 838         self.options = [
 839                 optparse.make_option("--origin", dest="origin"),
 840                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 841                 # preserve the user, requires relevant p4 permissions
 842                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 843                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 844         ]
 845         self.description = "Submit changes from git to the perforce depot."
 846         self.usage += " [name of git branch to submit into perforce depot]"
 847         self.interactive = True
 848         self.origin = ""
 849         self.detectRenames = False
 850         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 851         self.isWindows = (platform.system() == "Windows")
 852         self.exportLabels = False
 853
 854     def check(self):
 855         if len(p4CmdList("opened ...")) > 0:
 856             die("You have files opened with perforce! Close them before starting the sync.")
 857
 858     # replaces everything between 'Description:' and the next P4 submit template field with the
 859     # commit message
 860     def prepareLogMessage(self, template, message):
 861         result = ""
 862
 863         inDescriptionSection = False
 864
 865         for line in template.split("\n"):
 866             if line.startswith("#"):
 867                 result += line + "\n"
 868                 continue
 869
 870             if inDescriptionSection:
 871                 if line.startswith("Files:") or line.startswith("Jobs:"):
 872                     inDescriptionSection = False
 873                 else:
 874                     continue
 875             else:
 876                 if line.startswith("Description:"):
 877                     inDescriptionSection = True
 878                     line += "\n"
 879                     for messageLine in message.split("\n"):
 880                         line += "\t" + messageLine + "\n"
 881
 882             result += line + "\n"
 883
 884         return result
 885
 886     def patchRCSKeywords(self, file, pattern):
 887         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
 888         (handle, outFileName) = tempfile.mkstemp(dir='.')
 889         try:
 890             outFile = os.fdopen(handle, "w+")
 891             inFile = open(file, "r")
 892             regexp = re.compile(pattern, re.VERBOSE)
 893             for line in inFile.readlines():
 894                 line = regexp.sub(r'$\1$', line)
 895                 outFile.write(line)
 896             inFile.close()
 897             outFile.close()
 898             # Forcibly overwrite the original file
 899             os.unlink(file)
 900             shutil.move(outFileName, file)
 901         except:
 902             # cleanup our temporary file
 903             os.unlink(outFileName)
 904             print "Failed to strip RCS keywords in %s" % file
 905             raise
 906
 907         print "Patched up RCS keywords in %s" % file
 908
 909     def p4UserForCommit(self,id):
 910         # Return the tuple (perforce user,git email) for a given git commit id
 911         self.getUserMapFromPerforceServer()
 912         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 913         gitEmail = gitEmail.strip()
 914         if not self.emails.has_key(gitEmail):
 915             return (None,gitEmail)
 916         else:
 917             return (self.emails[gitEmail],gitEmail)
 918
 919     def checkValidP4Users(self,commits):
 920         # check if any git authors cannot be mapped to p4 users
 921         for id in commits:
 922             (user,email) = self.p4UserForCommit(id)
 923             if not user:
 924                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 925                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 926                     print "%s" % msg
 927                 else:
 928                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 929
 930     def lastP4Changelist(self):
 931         # Get back the last changelist number submitted in this client spec. This
 932         # then gets used to patch up the username in the change. If the same
 933         # client spec is being used by multiple processes then this might go
 934         # wrong.
 935         results = p4CmdList("client -o")        # find the current client
 936         client = None
 937         for r in results:
 938             if r.has_key('Client'):
 939                 client = r['Client']
 940                 break
 941         if not client:
 942             die("could not get client spec")
 943         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 944         for r in results:
 945             if r.has_key('change'):
 946                 return r['change']
 947         die("Could not get changelist number for last submit - cannot patch up user details")
 948
 949     def modifyChangelistUser(self, changelist, newUser):
 950         # fixup the user field of a changelist after it has been submitted.
 951         changes = p4CmdList("change -o %s" % changelist)
 952         if len(changes) != 1:
 953             die("Bad output from p4 change modifying %s to user %s" %
 954                 (changelist, newUser))
 955
 956         c = changes[0]
 957         if c['User'] == newUser: return   # nothing to do
 958         c['User'] = newUser
 959         input = marshal.dumps(c)
 960
 961         result = p4CmdList("change -f -i", stdin=input)
 962         for r in result:
 963             if r.has_key('code'):
 964                 if r['code'] == 'error':
 965                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 966             if r.has_key('data'):
 967                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 968                 return
 969         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 970
 971     def canChangeChangelists(self):
 972         # check to see if we have p4 admin or super-user permissions, either of
 973         # which are required to modify changelists.
 974         results = p4CmdList(["protects", self.depotPath])
 975         for r in results:
 976             if r.has_key('perm'):
 977                 if r['perm'] == 'admin':
 978                     return 1
 979                 if r['perm'] == 'super':
 980                     return 1
 981         return 0
 982
 983     def prepareSubmitTemplate(self):
 984         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 985         template = ""
 986         inFilesSection = False
 987         for line in p4_read_pipe_lines(['change', '-o']):
 988             if line.endswith("\r\n"):
 989                 line = line[:-2] + "\n"
 990             if inFilesSection:
 991                 if line.startswith("\t"):
 992                     # path starts and ends with a tab
 993                     path = line[1:]
 994                     lastTab = path.rfind("\t")
 995                     if lastTab != -1:
 996                         path = path[:lastTab]
 997                         if not p4PathStartsWith(path, self.depotPath):
 998                             continue
 999                 else:
1000                     inFilesSection = False
1001             else:
1002                 if line.startswith("Files:"):
1003                     inFilesSection = True
1004
1005             template += line
1006
1007         return template
1008
1009     def edit_template(self, template_file):
1010         """Invoke the editor to let the user change the submission
1011            message.  Return true if okay to continue with the submit."""
1012
1013         # if configured to skip the editing part, just submit
1014         if gitConfig("git-p4.skipSubmitEdit") == "true":
1015             return True
1016
1017         # look at the modification time, to check later if the user saved
1018         # the file
1019         mtime = os.stat(template_file).st_mtime
1020
1021         # invoke the editor
1022         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1023             editor = os.environ.get("P4EDITOR")
1024         else:
1025             editor = read_pipe("git var GIT_EDITOR").strip()
1026         system(editor + " " + template_file)
1027
1028         # If the file was not saved, prompt to see if this patch should
1029         # be skipped.  But skip this verification step if configured so.
1030         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1031             return True
1032
1033         # modification time updated means user saved the file
1034         if os.stat(template_file).st_mtime > mtime:
1035             return True
1036
1037         while True:
1038             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1039             if response == 'y':
1040                 return True
1041             if response == 'n':
1042                 return False
1043
1044     def applyCommit(self, id):
1045         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
1046
1047         (p4User, gitEmail) = self.p4UserForCommit(id)
1048
1049         if not self.detectRenames:
1050             # If not explicitly set check the config variable
1051             self.detectRenames = gitConfig("git-p4.detectRenames")
1052
1053         if self.detectRenames.lower() == "false" or self.detectRenames == "":
1054             diffOpts = ""
1055         elif self.detectRenames.lower() == "true":
1056             diffOpts = "-M"
1057         else:
1058             diffOpts = "-M%s" % self.detectRenames
1059
1060         detectCopies = gitConfig("git-p4.detectCopies")
1061         if detectCopies.lower() == "true":
1062             diffOpts += " -C"
1063         elif detectCopies != "" and detectCopies.lower() != "false":
1064             diffOpts += " -C%s" % detectCopies
1065
1066         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1067             diffOpts += " --find-copies-harder"
1068
1069         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
1070         filesToAdd = set()
1071         filesToDelete = set()
1072         editedFiles = set()
1073         pureRenameCopy = set()
1074         filesToChangeExecBit = {}
1075
1076         for line in diff:
1077             diff = parseDiffTreeEntry(line)
1078             modifier = diff['status']
1079             path = diff['src']
1080             if modifier == "M":
1081                 p4_edit(path)
1082                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1083                     filesToChangeExecBit[path] = diff['dst_mode']
1084                 editedFiles.add(path)
1085             elif modifier == "A":
1086                 filesToAdd.add(path)
1087                 filesToChangeExecBit[path] = diff['dst_mode']
1088                 if path in filesToDelete:
1089                     filesToDelete.remove(path)
1090             elif modifier == "D":
1091                 filesToDelete.add(path)
1092                 if path in filesToAdd:
1093                     filesToAdd.remove(path)
1094             elif modifier == "C":
1095                 src, dest = diff['src'], diff['dst']
1096                 p4_integrate(src, dest)
1097                 pureRenameCopy.add(dest)
1098                 if diff['src_sha1'] != diff['dst_sha1']:
1099                     p4_edit(dest)
1100                     pureRenameCopy.discard(dest)
1101                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1102                     p4_edit(dest)
1103                     pureRenameCopy.discard(dest)
1104                     filesToChangeExecBit[dest] = diff['dst_mode']
1105                 os.unlink(dest)
1106                 editedFiles.add(dest)
1107             elif modifier == "R":
1108                 src, dest = diff['src'], diff['dst']
1109                 p4_integrate(src, dest)
1110                 if diff['src_sha1'] != diff['dst_sha1']:
1111                     p4_edit(dest)
1112                 else:
1113                     pureRenameCopy.add(dest)
1114                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1115                     p4_edit(dest)
1116                     filesToChangeExecBit[dest] = diff['dst_mode']
1117                 os.unlink(dest)
1118                 editedFiles.add(dest)
1119                 filesToDelete.add(src)
1120             else:
1121                 die("unknown modifier %s for %s" % (modifier, path))
1122
1123         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1124         patchcmd = diffcmd + " | git apply "
1125         tryPatchCmd = patchcmd + "--check -"
1126         applyPatchCmd = patchcmd + "--check --apply -"
1127         patch_succeeded = True
1128
1129         if os.system(tryPatchCmd) != 0:
1130             fixed_rcs_keywords = False
1131             patch_succeeded = False
1132             print "Unfortunately applying the change failed!"
1133
1134             # Patch failed, maybe it's just RCS keyword woes. Look through
1135             # the patch to see if that's possible.
1136             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1137                 file = None
1138                 pattern = None
1139                 kwfiles = {}
1140                 for file in editedFiles | filesToDelete:
1141                     # did this file's delta contain RCS keywords?
1142                     pattern = p4_keywords_regexp_for_file(file)
1143
1144                     if pattern:
1145                         # this file is a possibility...look for RCS keywords.
1146                         regexp = re.compile(pattern, re.VERBOSE)
1147                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1148                             if regexp.search(line):
1149                                 if verbose:
1150                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1151                                 kwfiles[file] = pattern
1152                                 break
1153
1154                 for file in kwfiles:
1155                     if verbose:
1156                         print "zapping %s with %s" % (line,pattern)
1157                     self.patchRCSKeywords(file, kwfiles[file])
1158                     fixed_rcs_keywords = True
1159
1160             if fixed_rcs_keywords:
1161                 print "Retrying the patch with RCS keywords cleaned up"
1162                 if os.system(tryPatchCmd) == 0:
1163                     patch_succeeded = True
1164
1165         if not patch_succeeded:
1166             print "What do you want to do?"
1167             response = "x"
1168             while response != "s" and response != "a" and response != "w":
1169                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
1170                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
1171             if response == "s":
1172                 print "Skipping! Good luck with the next patches..."
1173                 for f in editedFiles:
1174                     p4_revert(f)
1175                 for f in filesToAdd:
1176                     os.remove(f)
1177                 return
1178             elif response == "a":
1179                 os.system(applyPatchCmd)
1180                 if len(filesToAdd) > 0:
1181                     print "You may also want to call p4 add on the following files:"
1182                     print " ".join(filesToAdd)
1183                 if len(filesToDelete):
1184                     print "The following files should be scheduled for deletion with p4 delete:"
1185                     print " ".join(filesToDelete)
1186                 die("Please resolve and submit the conflict manually and "
1187                     + "continue afterwards with git p4 submit --continue")
1188             elif response == "w":
1189                 system(diffcmd + " > patch.txt")
1190                 print "Patch saved to patch.txt in %s !" % self.clientPath
1191                 die("Please resolve and submit the conflict manually and "
1192                     "continue afterwards with git p4 submit --continue")
1193
1194         system(applyPatchCmd)
1195
1196         for f in filesToAdd:
1197             p4_add(f)
1198         for f in filesToDelete:
1199             p4_revert(f)
1200             p4_delete(f)
1201
1202         # Set/clear executable bits
1203         for f in filesToChangeExecBit.keys():
1204             mode = filesToChangeExecBit[f]
1205             setP4ExecBit(f, mode)
1206
1207         logMessage = extractLogMessageFromGitCommit(id)
1208         logMessage = logMessage.strip()
1209
1210         template = self.prepareSubmitTemplate()
1211
1212         if self.interactive:
1213             submitTemplate = self.prepareLogMessage(template, logMessage)
1214
1215             if self.preserveUser:
1216                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1217
1218             if os.environ.has_key("P4DIFF"):
1219                 del(os.environ["P4DIFF"])
1220             diff = ""
1221             for editedFile in editedFiles:
1222                 diff += p4_read_pipe(['diff', '-du',
1223                                       wildcard_encode(editedFile)])
1224
1225             newdiff = ""
1226             for newFile in filesToAdd:
1227                 newdiff += "==== new file ====\n"
1228                 newdiff += "--- /dev/null\n"
1229                 newdiff += "+++ %s\n" % newFile
1230                 f = open(newFile, "r")
1231                 for line in f.readlines():
1232                     newdiff += "+" + line
1233                 f.close()
1234
1235             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1236                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1237                 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1238                 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1239
1240             separatorLine = "######## everything below this line is just the diff #######\n"
1241
1242             (handle, fileName) = tempfile.mkstemp()
1243             tmpFile = os.fdopen(handle, "w+")
1244             if self.isWindows:
1245                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1246                 separatorLine = separatorLine.replace("\n", "\r\n")
1247                 newdiff = newdiff.replace("\n", "\r\n")
1248             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1249             tmpFile.close()
1250
1251             if self.edit_template(fileName):
1252                 # read the edited message and submit
1253                 tmpFile = open(fileName, "rb")
1254                 message = tmpFile.read()
1255                 tmpFile.close()
1256                 submitTemplate = message[:message.index(separatorLine)]
1257                 if self.isWindows:
1258                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1259                 p4_write_pipe(['submit', '-i'], submitTemplate)
1260
1261                 if self.preserveUser:
1262                     if p4User:
1263                         # Get last changelist number. Cannot easily get it from
1264                         # the submit command output as the output is
1265                         # unmarshalled.
1266                         changelist = self.lastP4Changelist()
1267                         self.modifyChangelistUser(changelist, p4User)
1268
1269                 # The rename/copy happened by applying a patch that created a
1270                 # new file.  This leaves it writable, which confuses p4.
1271                 for f in pureRenameCopy:
1272                     p4_sync(f, "-f")
1273
1274             else:
1275                 # skip this patch
1276                 print "Submission cancelled, undoing p4 changes."
1277                 for f in editedFiles:
1278                     p4_revert(f)
1279                 for f in filesToAdd:
1280                     p4_revert(f)
1281                     os.remove(f)
1282
1283             os.remove(fileName)
1284         else:
1285             fileName = "submit.txt"
1286             file = open(fileName, "w+")
1287             file.write(self.prepareLogMessage(template, logMessage))
1288             file.close()
1289             print ("Perforce submit template written as %s. "
1290                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1291                    % (fileName, fileName))
1292
1293     # Export git tags as p4 labels. Create a p4 label and then tag
1294     # with that.
1295     def exportGitTags(self, gitTags):
1296         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1297         if len(validLabelRegexp) == 0:
1298             validLabelRegexp = defaultLabelRegexp
1299         m = re.compile(validLabelRegexp)
1300
1301         for name in gitTags:
1302
1303             if not m.match(name):
1304                 if verbose:
1305                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1306                 continue
1307
1308             # Get the p4 commit this corresponds to
1309             logMessage = extractLogMessageFromGitCommit(name)
1310             values = extractSettingsGitLog(logMessage)
1311
1312             if not values.has_key('change'):
1313                 # a tag pointing to something not sent to p4; ignore
1314                 if verbose:
1315                     print "git tag %s does not give a p4 commit" % name
1316                 continue
1317             else:
1318                 changelist = values['change']
1319
1320             # Get the tag details.
1321             inHeader = True
1322             isAnnotated = False
1323             body = []
1324             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1325                 l = l.strip()
1326                 if inHeader:
1327                     if re.match(r'tag\s+', l):
1328                         isAnnotated = True
1329                     elif re.match(r'\s*$', l):
1330                         inHeader = False
1331                         continue
1332                 else:
1333                     body.append(l)
1334
1335             if not isAnnotated:
1336                 body = ["lightweight tag imported by git p4\n"]
1337
1338             # Create the label - use the same view as the client spec we are using
1339             clientSpec = getClientSpec()
1340
1341             labelTemplate  = "Label: %s\n" % name
1342             labelTemplate += "Description:\n"
1343             for b in body:
1344                 labelTemplate += "\t" + b + "\n"
1345             labelTemplate += "View:\n"
1346             for mapping in clientSpec.mappings:
1347                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1348
1349             p4_write_pipe(["label", "-i"], labelTemplate)
1350
1351             # Use the label
1352             p4_system(["tag", "-l", name] +
1353                       ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1354
1355             if verbose:
1356                 print "created p4 label for tag %s" % name
1357
1358     def run(self, args):
1359         if len(args) == 0:
1360             self.master = currentGitBranch()
1361             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1362                 die("Detecting current git branch failed!")
1363         elif len(args) == 1:
1364             self.master = args[0]
1365             if not branchExists(self.master):
1366                 die("Branch %s does not exist" % self.master)
1367         else:
1368             return False
1369
1370         allowSubmit = gitConfig("git-p4.allowSubmit")
1371         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1372             die("%s is not in git-p4.allowSubmit" % self.master)
1373
1374         [upstream, settings] = findUpstreamBranchPoint()
1375         self.depotPath = settings['depot-paths'][0]
1376         if len(self.origin) == 0:
1377             self.origin = upstream
1378
1379         if self.preserveUser:
1380             if not self.canChangeChangelists():
1381                 die("Cannot preserve user names without p4 super-user or admin permissions")
1382
1383         if self.verbose:
1384             print "Origin branch is " + self.origin
1385
1386         if len(self.depotPath) == 0:
1387             print "Internal error: cannot locate perforce depot path from existing branches"
1388             sys.exit(128)
1389
1390         self.useClientSpec = False
1391         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1392             self.useClientSpec = True
1393         if self.useClientSpec:
1394             self.clientSpecDirs = getClientSpec()
1395
1396         if self.useClientSpec:
1397             # all files are relative to the client spec
1398             self.clientPath = getClientRoot()
1399         else:
1400             self.clientPath = p4Where(self.depotPath)
1401
1402         if self.clientPath == "":
1403             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1404
1405         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1406         self.oldWorkingDirectory = os.getcwd()
1407
1408         # ensure the clientPath exists
1409         new_client_dir = False
1410         if not os.path.exists(self.clientPath):
1411             new_client_dir = True
1412             os.makedirs(self.clientPath)
1413
1414         chdir(self.clientPath)
1415         print "Synchronizing p4 checkout..."
1416         if new_client_dir:
1417             # old one was destroyed, and maybe nobody told p4
1418             p4_sync("...", "-f")
1419         else:
1420             p4_sync("...")
1421         self.check()
1422
1423         commits = []
1424         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1425             commits.append(line.strip())
1426         commits.reverse()
1427
1428         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1429             self.checkAuthorship = False
1430         else:
1431             self.checkAuthorship = True
1432
1433         if self.preserveUser:
1434             self.checkValidP4Users(commits)
1435
1436         while len(commits) > 0:
1437             commit = commits[0]
1438             commits = commits[1:]
1439             self.applyCommit(commit)
1440             if not self.interactive:
1441                 break
1442
1443         if len(commits) == 0:
1444             print "All changes applied!"
1445             chdir(self.oldWorkingDirectory)
1446
1447             sync = P4Sync()
1448             sync.run([])
1449
1450             rebase = P4Rebase()
1451             rebase.rebase()
1452
1453         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1454             self.exportLabels = True
1455
1456         if self.exportLabels:
1457             p4Labels = getP4Labels(self.depotPath)
1458             gitTags = getGitTags()
1459
1460             missingGitTags = gitTags - p4Labels
1461             self.exportGitTags(missingGitTags)
1462
1463         return True
1464
1465 class View(object):
1466     """Represent a p4 view ("p4 help views"), and map files in a
1467        repo according to the view."""
1468
1469     class Path(object):
1470         """A depot or client path, possibly containing wildcards.
1471            The only one supported is ... at the end, currently.
1472            Initialize with the full path, with //depot or //client."""
1473
1474         def __init__(self, path, is_depot):
1475             self.path = path
1476             self.is_depot = is_depot
1477             self.find_wildcards()
1478             # remember the prefix bit, useful for relative mappings
1479             m = re.match("(//[^/]+/)", self.path)
1480             if not m:
1481                 die("Path %s does not start with //prefix/" % self.path)
1482             prefix = m.group(1)
1483             if not self.is_depot:
1484                 # strip //client/ on client paths
1485                 self.path = self.path[len(prefix):]
1486
1487         def find_wildcards(self):
1488             """Make sure wildcards are valid, and set up internal
1489                variables."""
1490
1491             self.ends_triple_dot = False
1492             # There are three wildcards allowed in p4 views
1493             # (see "p4 help views").  This code knows how to
1494             # handle "..." (only at the end), but cannot deal with
1495             # "%%n" or "*".  Only check the depot_side, as p4 should
1496             # validate that the client_side matches too.
1497             if re.search(r'%%[1-9]', self.path):
1498                 die("Can't handle %%n wildcards in view: %s" % self.path)
1499             if self.path.find("*") >= 0:
1500                 die("Can't handle * wildcards in view: %s" % self.path)
1501             triple_dot_index = self.path.find("...")
1502             if triple_dot_index >= 0:
1503                 if triple_dot_index != len(self.path) - 3:
1504                     die("Can handle only single ... wildcard, at end: %s" %
1505                         self.path)
1506                 self.ends_triple_dot = True
1507
1508         def ensure_compatible(self, other_path):
1509             """Make sure the wildcards agree."""
1510             if self.ends_triple_dot != other_path.ends_triple_dot:
1511                  die("Both paths must end with ... if either does;\n" +
1512                      "paths: %s %s" % (self.path, other_path.path))
1513
1514         def match_wildcards(self, test_path):
1515             """See if this test_path matches us, and fill in the value
1516                of the wildcards if so.  Returns a tuple of
1517                (True|False, wildcards[]).  For now, only the ... at end
1518                is supported, so at most one wildcard."""
1519             if self.ends_triple_dot:
1520                 dotless = self.path[:-3]
1521                 if test_path.startswith(dotless):
1522                     wildcard = test_path[len(dotless):]
1523                     return (True, [ wildcard ])
1524             else:
1525                 if test_path == self.path:
1526                     return (True, [])
1527             return (False, [])
1528
1529         def match(self, test_path):
1530             """Just return if it matches; don't bother with the wildcards."""
1531             b, _ = self.match_wildcards(test_path)
1532             return b
1533
1534         def fill_in_wildcards(self, wildcards):
1535             """Return the relative path, with the wildcards filled in
1536                if there are any."""
1537             if self.ends_triple_dot:
1538                 return self.path[:-3] + wildcards[0]
1539             else:
1540                 return self.path
1541
1542     class Mapping(object):
1543         def __init__(self, depot_side, client_side, overlay, exclude):
1544             # depot_side is without the trailing /... if it had one
1545             self.depot_side = View.Path(depot_side, is_depot=True)
1546             self.client_side = View.Path(client_side, is_depot=False)
1547             self.overlay = overlay  # started with "+"
1548             self.exclude = exclude  # started with "-"
1549             assert not (self.overlay and self.exclude)
1550             self.depot_side.ensure_compatible(self.client_side)
1551
1552         def __str__(self):
1553             c = " "
1554             if self.overlay:
1555                 c = "+"
1556             if self.exclude:
1557                 c = "-"
1558             return "View.Mapping: %s%s -> %s" % \
1559                    (c, self.depot_side.path, self.client_side.path)
1560
1561         def map_depot_to_client(self, depot_path):
1562             """Calculate the client path if using this mapping on the
1563                given depot path; does not consider the effect of other
1564                mappings in a view.  Even excluded mappings are returned."""
1565             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1566             if not matches:
1567                 return ""
1568             client_path = self.client_side.fill_in_wildcards(wildcards)
1569             return client_path
1570
1571     #
1572     # View methods
1573     #
1574     def __init__(self):
1575         self.mappings = []
1576
1577     def append(self, view_line):
1578         """Parse a view line, splitting it into depot and client
1579            sides.  Append to self.mappings, preserving order."""
1580
1581         # Split the view line into exactly two words.  P4 enforces
1582         # structure on these lines that simplifies this quite a bit.
1583         #
1584         # Either or both words may be double-quoted.
1585         # Single quotes do not matter.
1586         # Double-quote marks cannot occur inside the words.
1587         # A + or - prefix is also inside the quotes.
1588         # There are no quotes unless they contain a space.
1589         # The line is already white-space stripped.
1590         # The two words are separated by a single space.
1591         #
1592         if view_line[0] == '"':
1593             # First word is double quoted.  Find its end.
1594             close_quote_index = view_line.find('"', 1)
1595             if close_quote_index <= 0:
1596                 die("No first-word closing quote found: %s" % view_line)
1597             depot_side = view_line[1:close_quote_index]
1598             # skip closing quote and space
1599             rhs_index = close_quote_index + 1 + 1
1600         else:
1601             space_index = view_line.find(" ")
1602             if space_index <= 0:
1603                 die("No word-splitting space found: %s" % view_line)
1604             depot_side = view_line[0:space_index]
1605             rhs_index = space_index + 1
1606
1607         if view_line[rhs_index] == '"':
1608             # Second word is double quoted.  Make sure there is a
1609             # double quote at the end too.
1610             if not view_line.endswith('"'):
1611                 die("View line with rhs quote should end with one: %s" %
1612                     view_line)
1613             # skip the quotes
1614             client_side = view_line[rhs_index+1:-1]
1615         else:
1616             client_side = view_line[rhs_index:]
1617
1618         # prefix + means overlay on previous mapping
1619         overlay = False
1620         if depot_side.startswith("+"):
1621             overlay = True
1622             depot_side = depot_side[1:]
1623
1624         # prefix - means exclude this path
1625         exclude = False
1626         if depot_side.startswith("-"):
1627             exclude = True
1628             depot_side = depot_side[1:]
1629
1630         m = View.Mapping(depot_side, client_side, overlay, exclude)
1631         self.mappings.append(m)
1632
1633     def map_in_client(self, depot_path):
1634         """Return the relative location in the client where this
1635            depot file should live.  Returns "" if the file should
1636            not be mapped in the client."""
1637
1638         paths_filled = []
1639         client_path = ""
1640
1641         # look at later entries first
1642         for m in self.mappings[::-1]:
1643
1644             # see where will this path end up in the client
1645             p = m.map_depot_to_client(depot_path)
1646
1647             if p == "":
1648                 # Depot path does not belong in client.  Must remember
1649                 # this, as previous items should not cause files to
1650                 # exist in this path either.  Remember that the list is
1651                 # being walked from the end, which has higher precedence.
1652                 # Overlap mappings do not exclude previous mappings.
1653                 if not m.overlay:
1654                     paths_filled.append(m.client_side)
1655
1656             else:
1657                 # This mapping matched; no need to search any further.
1658                 # But, the mapping could be rejected if the client path
1659                 # has already been claimed by an earlier mapping (i.e.
1660                 # one later in the list, which we are walking backwards).
1661                 already_mapped_in_client = False
1662                 for f in paths_filled:
1663                     # this is View.Path.match
1664                     if f.match(p):
1665                         already_mapped_in_client = True
1666                         break
1667                 if not already_mapped_in_client:
1668                     # Include this file, unless it is from a line that
1669                     # explicitly said to exclude it.
1670                     if not m.exclude:
1671                         client_path = p
1672
1673                 # a match, even if rejected, always stops the search
1674                 break
1675
1676         return client_path
1677
1678 class P4Sync(Command, P4UserMap):
1679     delete_actions = ( "delete", "move/delete", "purge" )
1680
1681     def __init__(self):
1682         Command.__init__(self)
1683         P4UserMap.__init__(self)
1684         self.options = [
1685                 optparse.make_option("--branch", dest="branch"),
1686                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1687                 optparse.make_option("--changesfile", dest="changesFile"),
1688                 optparse.make_option("--silent", dest="silent", action="store_true"),
1689                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1690                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1691                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1692                                      help="Import into refs/heads/ , not refs/remotes"),
1693                 optparse.make_option("--max-changes", dest="maxChanges"),
1694                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1695                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1696                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1697                                      help="Only sync files that are included in the Perforce Client Spec")
1698         ]
1699         self.description = """Imports from Perforce into a git repository.\n
1700     example:
1701     //depot/my/project/ -- to import the current head
1702     //depot/my/project/@all -- to import everything
1703     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1704
1705     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1706
1707         self.usage += " //depot/path[@revRange]"
1708         self.silent = False
1709         self.createdBranches = set()
1710         self.committedChanges = set()
1711         self.branch = ""
1712         self.detectBranches = False
1713         self.detectLabels = False
1714         self.importLabels = False
1715         self.changesFile = ""
1716         self.syncWithOrigin = True
1717         self.importIntoRemotes = True
1718         self.maxChanges = ""
1719         self.isWindows = (platform.system() == "Windows")
1720         self.keepRepoPath = False
1721         self.depotPaths = None
1722         self.p4BranchesInGit = []
1723         self.cloneExclude = []
1724         self.useClientSpec = False
1725         self.useClientSpec_from_options = False
1726         self.clientSpecDirs = None
1727         self.tempBranches = []
1728         self.tempBranchLocation = "git-p4-tmp"
1729
1730         if gitConfig("git-p4.syncFromOrigin") == "false":
1731             self.syncWithOrigin = False
1732
1733     # Force a checkpoint in fast-import and wait for it to finish
1734     def checkpoint(self):
1735         self.gitStream.write("checkpoint\n\n")
1736         self.gitStream.write("progress checkpoint\n\n")
1737         out = self.gitOutput.readline()
1738         if self.verbose:
1739             print "checkpoint finished: " + out
1740
1741     def extractFilesFromCommit(self, commit):
1742         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1743                              for path in self.cloneExclude]
1744         files = []
1745         fnum = 0
1746         while commit.has_key("depotFile%s" % fnum):
1747             path =  commit["depotFile%s" % fnum]
1748
1749             if [p for p in self.cloneExclude
1750                 if p4PathStartsWith(path, p)]:
1751                 found = False
1752             else:
1753                 found = [p for p in self.depotPaths
1754                          if p4PathStartsWith(path, p)]
1755             if not found:
1756                 fnum = fnum + 1
1757                 continue
1758
1759             file = {}
1760             file["path"] = path
1761             file["rev"] = commit["rev%s" % fnum]
1762             file["action"] = commit["action%s" % fnum]
1763             file["type"] = commit["type%s" % fnum]
1764             files.append(file)
1765             fnum = fnum + 1
1766         return files
1767
1768     def stripRepoPath(self, path, prefixes):
1769         if self.useClientSpec:
1770             return self.clientSpecDirs.map_in_client(path)
1771
1772         if self.keepRepoPath:
1773             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1774
1775         for p in prefixes:
1776             if p4PathStartsWith(path, p):
1777                 path = path[len(p):]
1778
1779         return path
1780
1781     def splitFilesIntoBranches(self, commit):
1782         branches = {}
1783         fnum = 0
1784         while commit.has_key("depotFile%s" % fnum):
1785             path =  commit["depotFile%s" % fnum]
1786             found = [p for p in self.depotPaths
1787                      if p4PathStartsWith(path, p)]
1788             if not found:
1789                 fnum = fnum + 1
1790                 continue
1791
1792             file = {}
1793             file["path"] = path
1794             file["rev"] = commit["rev%s" % fnum]
1795             file["action"] = commit["action%s" % fnum]
1796             file["type"] = commit["type%s" % fnum]
1797             fnum = fnum + 1
1798
1799             relPath = self.stripRepoPath(path, self.depotPaths)
1800             relPath = wildcard_decode(relPath)
1801
1802             for branch in self.knownBranches.keys():
1803
1804                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1805                 if relPath.startswith(branch + "/"):
1806                     if branch not in branches:
1807                         branches[branch] = []
1808                     branches[branch].append(file)
1809                     break
1810
1811         return branches
1812
1813     # output one file from the P4 stream
1814     # - helper for streamP4Files
1815
1816     def streamOneP4File(self, file, contents):
1817         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1818         relPath = wildcard_decode(relPath)
1819         if verbose:
1820             sys.stderr.write("%s\n" % relPath)
1821
1822         (type_base, type_mods) = split_p4_type(file["type"])
1823
1824         git_mode = "100644"
1825         if "x" in type_mods:
1826             git_mode = "100755"
1827         if type_base == "symlink":
1828             git_mode = "120000"
1829             # p4 print on a symlink contains "target\n"; remove the newline
1830             data = ''.join(contents)
1831             contents = [data[:-1]]
1832
1833         if type_base == "utf16":
1834             # p4 delivers different text in the python output to -G
1835             # than it does when using "print -o", or normal p4 client
1836             # operations.  utf16 is converted to ascii or utf8, perhaps.
1837             # But ascii text saved as -t utf16 is completely mangled.
1838             # Invoke print -o to get the real contents.
1839             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1840             contents = [ text ]
1841
1842         if type_base == "apple":
1843             # Apple filetype files will be streamed as a concatenation of
1844             # its appledouble header and the contents.  This is useless
1845             # on both macs and non-macs.  If using "print -q -o xx", it
1846             # will create "xx" with the data, and "%xx" with the header.
1847             # This is also not very useful.
1848             #
1849             # Ideally, someday, this script can learn how to generate
1850             # appledouble files directly and import those to git, but
1851             # non-mac machines can never find a use for apple filetype.
1852             print "\nIgnoring apple filetype file %s" % file['depotFile']
1853             return
1854
1855         # Perhaps windows wants unicode, utf16 newlines translated too;
1856         # but this is not doing it.
1857         if self.isWindows and type_base == "text":
1858             mangled = []
1859             for data in contents:
1860                 data = data.replace("\r\n", "\n")
1861                 mangled.append(data)
1862             contents = mangled
1863
1864         # Note that we do not try to de-mangle keywords on utf16 files,
1865         # even though in theory somebody may want that.
1866         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1867         if pattern:
1868             regexp = re.compile(pattern, re.VERBOSE)
1869             text = ''.join(contents)
1870             text = regexp.sub(r'$\1$', text)
1871             contents = [ text ]
1872
1873         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1874
1875         # total length...
1876         length = 0
1877         for d in contents:
1878             length = length + len(d)
1879
1880         self.gitStream.write("data %d\n" % length)
1881         for d in contents:
1882             self.gitStream.write(d)
1883         self.gitStream.write("\n")
1884
1885     def streamOneP4Deletion(self, file):
1886         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1887         relPath = wildcard_decode(relPath)
1888         if verbose:
1889             sys.stderr.write("delete %s\n" % relPath)
1890         self.gitStream.write("D %s\n" % relPath)
1891
1892     # handle another chunk of streaming data
1893     def streamP4FilesCb(self, marshalled):
1894
1895         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1896             # start of a new file - output the old one first
1897             self.streamOneP4File(self.stream_file, self.stream_contents)
1898             self.stream_file = {}
1899             self.stream_contents = []
1900             self.stream_have_file_info = False
1901
1902         # pick up the new file information... for the
1903         # 'data' field we need to append to our array
1904         for k in marshalled.keys():
1905             if k == 'data':
1906                 self.stream_contents.append(marshalled['data'])
1907             else:
1908                 self.stream_file[k] = marshalled[k]
1909
1910         self.stream_have_file_info = True
1911
1912     # Stream directly from "p4 files" into "git fast-import"
1913     def streamP4Files(self, files):
1914         filesForCommit = []
1915         filesToRead = []
1916         filesToDelete = []
1917
1918         for f in files:
1919             # if using a client spec, only add the files that have
1920             # a path in the client
1921             if self.clientSpecDirs:
1922                 if self.clientSpecDirs.map_in_client(f['path']) == "":
1923                     continue
1924
1925             filesForCommit.append(f)
1926             if f['action'] in self.delete_actions:
1927                 filesToDelete.append(f)
1928             else:
1929                 filesToRead.append(f)
1930
1931         # deleted files...
1932         for f in filesToDelete:
1933             self.streamOneP4Deletion(f)
1934
1935         if len(filesToRead) > 0:
1936             self.stream_file = {}
1937             self.stream_contents = []
1938             self.stream_have_file_info = False
1939
1940             # curry self argument
1941             def streamP4FilesCbSelf(entry):
1942                 self.streamP4FilesCb(entry)
1943
1944             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1945
1946             p4CmdList(["-x", "-", "print"],
1947                       stdin=fileArgs,
1948                       cb=streamP4FilesCbSelf)
1949
1950             # do the last chunk
1951             if self.stream_file.has_key('depotFile'):
1952                 self.streamOneP4File(self.stream_file, self.stream_contents)
1953
1954     def make_email(self, userid):
1955         if userid in self.users:
1956             return self.users[userid]
1957         else:
1958             return "%s <a@b>" % userid
1959
1960     # Stream a p4 tag
1961     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
1962         if verbose:
1963             print "writing tag %s for commit %s" % (labelName, commit)
1964         gitStream.write("tag %s\n" % labelName)
1965         gitStream.write("from %s\n" % commit)
1966
1967         if labelDetails.has_key('Owner'):
1968             owner = labelDetails["Owner"]
1969         else:
1970             owner = None
1971
1972         # Try to use the owner of the p4 label, or failing that,
1973         # the current p4 user id.
1974         if owner:
1975             email = self.make_email(owner)
1976         else:
1977             email = self.make_email(self.p4UserId())
1978         tagger = "%s %s %s" % (email, epoch, self.tz)
1979
1980         gitStream.write("tagger %s\n" % tagger)
1981
1982         print "labelDetails=",labelDetails
1983         if labelDetails.has_key('Description'):
1984             description = labelDetails['Description']
1985         else:
1986             description = 'Label from git p4'
1987
1988         gitStream.write("data %d\n" % len(description))
1989         gitStream.write(description)
1990         gitStream.write("\n")
1991
1992     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1993         epoch = details["time"]
1994         author = details["user"]
1995         self.branchPrefixes = branchPrefixes
1996
1997         if self.verbose:
1998             print "commit into %s" % branch
1999
2000         # start with reading files; if that fails, we should not
2001         # create a commit.
2002         new_files = []
2003         for f in files:
2004             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
2005                 new_files.append (f)
2006             else:
2007                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2008
2009         self.gitStream.write("commit %s\n" % branch)
2010 #        gitStream.write("mark :%s\n" % details["change"])
2011         self.committedChanges.add(int(details["change"]))
2012         committer = ""
2013         if author not in self.users:
2014             self.getUserMapFromPerforceServer()
2015         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2016
2017         self.gitStream.write("committer %s\n" % committer)
2018
2019         self.gitStream.write("data <<EOT\n")
2020         self.gitStream.write(details["desc"])
2021         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
2022                              % (','.join (branchPrefixes), details["change"]))
2023         if len(details['options']) > 0:
2024             self.gitStream.write(": options = %s" % details['options'])
2025         self.gitStream.write("]\nEOT\n\n")
2026
2027         if len(parent) > 0:
2028             if self.verbose:
2029                 print "parent %s" % parent
2030             self.gitStream.write("from %s\n" % parent)
2031
2032         self.streamP4Files(new_files)
2033         self.gitStream.write("\n")
2034
2035         change = int(details["change"])
2036
2037         if self.labels.has_key(change):
2038             label = self.labels[change]
2039             labelDetails = label[0]
2040             labelRevisions = label[1]
2041             if self.verbose:
2042                 print "Change %s is labelled %s" % (change, labelDetails)
2043
2044             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2045                                                     for p in branchPrefixes])
2046
2047             if len(files) == len(labelRevisions):
2048
2049                 cleanedFiles = {}
2050                 for info in files:
2051                     if info["action"] in self.delete_actions:
2052                         continue
2053                     cleanedFiles[info["depotFile"]] = info["rev"]
2054
2055                 if cleanedFiles == labelRevisions:
2056                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2057
2058                 else:
2059                     if not self.silent:
2060                         print ("Tag %s does not match with change %s: files do not match."
2061                                % (labelDetails["label"], change))
2062
2063             else:
2064                 if not self.silent:
2065                     print ("Tag %s does not match with change %s: file count is different."
2066                            % (labelDetails["label"], change))
2067
2068     # Build a dictionary of changelists and labels, for "detect-labels" option.
2069     def getLabels(self):
2070         self.labels = {}
2071
2072         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2073         if len(l) > 0 and not self.silent:
2074             print "Finding files belonging to labels in %s" % `self.depotPaths`
2075
2076         for output in l:
2077             label = output["label"]
2078             revisions = {}
2079             newestChange = 0
2080             if self.verbose:
2081                 print "Querying files for label %s" % label
2082             for file in p4CmdList(["files"] +
2083                                       ["%s...@%s" % (p, label)
2084                                           for p in self.depotPaths]):
2085                 revisions[file["depotFile"]] = file["rev"]
2086                 change = int(file["change"])
2087                 if change > newestChange:
2088                     newestChange = change
2089
2090             self.labels[newestChange] = [output, revisions]
2091
2092         if self.verbose:
2093             print "Label changes: %s" % self.labels.keys()
2094
2095     # Import p4 labels as git tags. A direct mapping does not
2096     # exist, so assume that if all the files are at the same revision
2097     # then we can use that, or it's something more complicated we should
2098     # just ignore.
2099     def importP4Labels(self, stream, p4Labels):
2100         if verbose:
2101             print "import p4 labels: " + ' '.join(p4Labels)
2102
2103         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2104         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2105         if len(validLabelRegexp) == 0:
2106             validLabelRegexp = defaultLabelRegexp
2107         m = re.compile(validLabelRegexp)
2108
2109         for name in p4Labels:
2110             commitFound = False
2111
2112             if not m.match(name):
2113                 if verbose:
2114                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2115                 continue
2116
2117             if name in ignoredP4Labels:
2118                 continue
2119
2120             labelDetails = p4CmdList(['label', "-o", name])[0]
2121
2122             # get the most recent changelist for each file in this label
2123             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2124                                 for p in self.depotPaths])
2125
2126             if change.has_key('change'):
2127                 # find the corresponding git commit; take the oldest commit
2128                 changelist = int(change['change'])
2129                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2130                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2131                 if len(gitCommit) == 0:
2132                     print "could not find git commit for changelist %d" % changelist
2133                 else:
2134                     gitCommit = gitCommit.strip()
2135                     commitFound = True
2136                     # Convert from p4 time format
2137                     try:
2138                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2139                     except ValueError:
2140                         print "Could not convert label time %s" % labelDetail['Update']
2141                         tmwhen = 1
2142
2143                     when = int(time.mktime(tmwhen))
2144                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2145                     if verbose:
2146                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2147             else:
2148                 if verbose:
2149                     print "Label %s has no changelists - possibly deleted?" % name
2150
2151             if not commitFound:
2152                 # We can't import this label; don't try again as it will get very
2153                 # expensive repeatedly fetching all the files for labels that will
2154                 # never be imported. If the label is moved in the future, the
2155                 # ignore will need to be removed manually.
2156                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2157
2158     def guessProjectName(self):
2159         for p in self.depotPaths:
2160             if p.endswith("/"):
2161                 p = p[:-1]
2162             p = p[p.strip().rfind("/") + 1:]
2163             if not p.endswith("/"):
2164                p += "/"
2165             return p
2166
2167     def getBranchMapping(self):
2168         lostAndFoundBranches = set()
2169
2170         user = gitConfig("git-p4.branchUser")
2171         if len(user) > 0:
2172             command = "branches -u %s" % user
2173         else:
2174             command = "branches"
2175
2176         for info in p4CmdList(command):
2177             details = p4Cmd(["branch", "-o", info["branch"]])
2178             viewIdx = 0
2179             while details.has_key("View%s" % viewIdx):
2180                 paths = details["View%s" % viewIdx].split(" ")
2181                 viewIdx = viewIdx + 1
2182                 # require standard //depot/foo/... //depot/bar/... mapping
2183                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2184                     continue
2185                 source = paths[0]
2186                 destination = paths[1]
2187                 ## HACK
2188                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2189                     source = source[len(self.depotPaths[0]):-4]
2190                     destination = destination[len(self.depotPaths[0]):-4]
2191
2192                     if destination in self.knownBranches:
2193                         if not self.silent:
2194                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2195                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2196                         continue
2197
2198                     self.knownBranches[destination] = source
2199
2200                     lostAndFoundBranches.discard(destination)
2201
2202                     if source not in self.knownBranches:
2203                         lostAndFoundBranches.add(source)
2204
2205         # Perforce does not strictly require branches to be defined, so we also
2206         # check git config for a branch list.
2207         #
2208         # Example of branch definition in git config file:
2209         # [git-p4]
2210         #   branchList=main:branchA
2211         #   branchList=main:branchB
2212         #   branchList=branchA:branchC
2213         configBranches = gitConfigList("git-p4.branchList")
2214         for branch in configBranches:
2215             if branch:
2216                 (source, destination) = branch.split(":")
2217                 self.knownBranches[destination] = source
2218
2219                 lostAndFoundBranches.discard(destination)
2220
2221                 if source not in self.knownBranches:
2222                     lostAndFoundBranches.add(source)
2223
2224
2225         for branch in lostAndFoundBranches:
2226             self.knownBranches[branch] = branch
2227
2228     def getBranchMappingFromGitBranches(self):
2229         branches = p4BranchesInGit(self.importIntoRemotes)
2230         for branch in branches.keys():
2231             if branch == "master":
2232                 branch = "main"
2233             else:
2234                 branch = branch[len(self.projectName):]
2235             self.knownBranches[branch] = branch
2236
2237     def listExistingP4GitBranches(self):
2238         # branches holds mapping from name to commit
2239         branches = p4BranchesInGit(self.importIntoRemotes)
2240         self.p4BranchesInGit = branches.keys()
2241         for branch in branches.keys():
2242             self.initialParents[self.refPrefix + branch] = branches[branch]
2243
2244     def updateOptionDict(self, d):
2245         option_keys = {}
2246         if self.keepRepoPath:
2247             option_keys['keepRepoPath'] = 1
2248
2249         d["options"] = ' '.join(sorted(option_keys.keys()))
2250
2251     def readOptions(self, d):
2252         self.keepRepoPath = (d.has_key('options')
2253                              and ('keepRepoPath' in d['options']))
2254
2255     def gitRefForBranch(self, branch):
2256         if branch == "main":
2257             return self.refPrefix + "master"
2258
2259         if len(branch) <= 0:
2260             return branch
2261
2262         return self.refPrefix + self.projectName + branch
2263
2264     def gitCommitByP4Change(self, ref, change):
2265         if self.verbose:
2266             print "looking in ref " + ref + " for change %s using bisect..." % change
2267
2268         earliestCommit = ""
2269         latestCommit = parseRevision(ref)
2270
2271         while True:
2272             if self.verbose:
2273                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2274             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2275             if len(next) == 0:
2276                 if self.verbose:
2277                     print "argh"
2278                 return ""
2279             log = extractLogMessageFromGitCommit(next)
2280             settings = extractSettingsGitLog(log)
2281             currentChange = int(settings['change'])
2282             if self.verbose:
2283                 print "current change %s" % currentChange
2284
2285             if currentChange == change:
2286                 if self.verbose:
2287                     print "found %s" % next
2288                 return next
2289
2290             if currentChange < change:
2291                 earliestCommit = "^%s" % next
2292             else:
2293                 latestCommit = "%s" % next
2294
2295         return ""
2296
2297     def importNewBranch(self, branch, maxChange):
2298         # make fast-import flush all changes to disk and update the refs using the checkpoint
2299         # command so that we can try to find the branch parent in the git history
2300         self.gitStream.write("checkpoint\n\n");
2301         self.gitStream.flush();
2302         branchPrefix = self.depotPaths[0] + branch + "/"
2303         range = "@1,%s" % maxChange
2304         #print "prefix" + branchPrefix
2305         changes = p4ChangesForPaths([branchPrefix], range)
2306         if len(changes) <= 0:
2307             return False
2308         firstChange = changes[0]
2309         #print "first change in branch: %s" % firstChange
2310         sourceBranch = self.knownBranches[branch]
2311         sourceDepotPath = self.depotPaths[0] + sourceBranch
2312         sourceRef = self.gitRefForBranch(sourceBranch)
2313         #print "source " + sourceBranch
2314
2315         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2316         #print "branch parent: %s" % branchParentChange
2317         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2318         if len(gitParent) > 0:
2319             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2320             #print "parent git commit: %s" % gitParent
2321
2322         self.importChanges(changes)
2323         return True
2324
2325     def searchParent(self, parent, branch, target):
2326         parentFound = False
2327         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2328             blob = blob.strip()
2329             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2330                 parentFound = True
2331                 if self.verbose:
2332                     print "Found parent of %s in commit %s" % (branch, blob)
2333                 break
2334         if parentFound:
2335             return blob
2336         else:
2337             return None
2338
2339     def importChanges(self, changes):
2340         cnt = 1
2341         for change in changes:
2342             description = p4Cmd(["describe", str(change)])
2343             self.updateOptionDict(description)
2344
2345             if not self.silent:
2346                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2347                 sys.stdout.flush()
2348             cnt = cnt + 1
2349
2350             try:
2351                 if self.detectBranches:
2352                     branches = self.splitFilesIntoBranches(description)
2353                     for branch in branches.keys():
2354                         ## HACK  --hwn
2355                         branchPrefix = self.depotPaths[0] + branch + "/"
2356
2357                         parent = ""
2358
2359                         filesForCommit = branches[branch]
2360
2361                         if self.verbose:
2362                             print "branch is %s" % branch
2363
2364                         self.updatedBranches.add(branch)
2365
2366                         if branch not in self.createdBranches:
2367                             self.createdBranches.add(branch)
2368                             parent = self.knownBranches[branch]
2369                             if parent == branch:
2370                                 parent = ""
2371                             else:
2372                                 fullBranch = self.projectName + branch
2373                                 if fullBranch not in self.p4BranchesInGit:
2374                                     if not self.silent:
2375                                         print("\n    Importing new branch %s" % fullBranch);
2376                                     if self.importNewBranch(branch, change - 1):
2377                                         parent = ""
2378                                         self.p4BranchesInGit.append(fullBranch)
2379                                     if not self.silent:
2380                                         print("\n    Resuming with change %s" % change);
2381
2382                                 if self.verbose:
2383                                     print "parent determined through known branches: %s" % parent
2384
2385                         branch = self.gitRefForBranch(branch)
2386                         parent = self.gitRefForBranch(parent)
2387
2388                         if self.verbose:
2389                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2390
2391                         if len(parent) == 0 and branch in self.initialParents:
2392                             parent = self.initialParents[branch]
2393                             del self.initialParents[branch]
2394
2395                         blob = None
2396                         if len(parent) > 0:
2397                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2398                             if self.verbose:
2399                                 print "Creating temporary branch: " + tempBranch
2400                             self.commit(description, filesForCommit, tempBranch, [branchPrefix])
2401                             self.tempBranches.append(tempBranch)
2402                             self.checkpoint()
2403                             blob = self.searchParent(parent, branch, tempBranch)
2404                         if blob:
2405                             self.commit(description, filesForCommit, branch, [branchPrefix], blob)
2406                         else:
2407                             if self.verbose:
2408                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2409                             self.commit(description, filesForCommit, branch, [branchPrefix], parent)
2410                 else:
2411                     files = self.extractFilesFromCommit(description)
2412                     self.commit(description, files, self.branch, self.depotPaths,
2413                                 self.initialParent)
2414                     self.initialParent = ""
2415             except IOError:
2416                 print self.gitError.read()
2417                 sys.exit(1)
2418
2419     def importHeadRevision(self, revision):
2420         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2421
2422         details = {}
2423         details["user"] = "git perforce import user"
2424         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2425                            % (' '.join(self.depotPaths), revision))
2426         details["change"] = revision
2427         newestRevision = 0
2428
2429         fileCnt = 0
2430         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2431
2432         for info in p4CmdList(["files"] + fileArgs):
2433
2434             if 'code' in info and info['code'] == 'error':
2435                 sys.stderr.write("p4 returned an error: %s\n"
2436                                  % info['data'])
2437                 if info['data'].find("must refer to client") >= 0:
2438                     sys.stderr.write("This particular p4 error is misleading.\n")
2439                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2440                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2441                 sys.exit(1)
2442             if 'p4ExitCode' in info:
2443                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2444                 sys.exit(1)
2445
2446
2447             change = int(info["change"])
2448             if change > newestRevision:
2449                 newestRevision = change
2450
2451             if info["action"] in self.delete_actions:
2452                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2453                 #fileCnt = fileCnt + 1
2454                 continue
2455
2456             for prop in ["depotFile", "rev", "action", "type" ]:
2457                 details["%s%s" % (prop, fileCnt)] = info[prop]
2458
2459             fileCnt = fileCnt + 1
2460
2461         details["change"] = newestRevision
2462
2463         # Use time from top-most change so that all git p4 clones of
2464         # the same p4 repo have the same commit SHA1s.
2465         res = p4CmdList("describe -s %d" % newestRevision)
2466         newestTime = None
2467         for r in res:
2468             if r.has_key('time'):
2469                 newestTime = int(r['time'])
2470         if newestTime is None:
2471             die("\"describe -s\" on newest change %d did not give a time")
2472         details["time"] = newestTime
2473
2474         self.updateOptionDict(details)
2475         try:
2476             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
2477         except IOError:
2478             print "IO error with git fast-import. Is your git version recent enough?"
2479             print self.gitError.read()
2480
2481
2482     def run(self, args):
2483         self.depotPaths = []
2484         self.changeRange = ""
2485         self.initialParent = ""
2486         self.previousDepotPaths = []
2487
2488         # map from branch depot path to parent branch
2489         self.knownBranches = {}
2490         self.initialParents = {}
2491         self.hasOrigin = originP4BranchesExist()
2492         if not self.syncWithOrigin:
2493             self.hasOrigin = False
2494
2495         if self.importIntoRemotes:
2496             self.refPrefix = "refs/remotes/p4/"
2497         else:
2498             self.refPrefix = "refs/heads/p4/"
2499
2500         if self.syncWithOrigin and self.hasOrigin:
2501             if not self.silent:
2502                 print "Syncing with origin first by calling git fetch origin"
2503             system("git fetch origin")
2504
2505         if len(self.branch) == 0:
2506             self.branch = self.refPrefix + "master"
2507             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2508                 system("git update-ref %s refs/heads/p4" % self.branch)
2509                 system("git branch -D p4");
2510             # create it /after/ importing, when master exists
2511             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2512                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2513
2514         # accept either the command-line option, or the configuration variable
2515         if self.useClientSpec:
2516             # will use this after clone to set the variable
2517             self.useClientSpec_from_options = True
2518         else:
2519             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2520                 self.useClientSpec = True
2521         if self.useClientSpec:
2522             self.clientSpecDirs = getClientSpec()
2523
2524         # TODO: should always look at previous commits,
2525         # merge with previous imports, if possible.
2526         if args == []:
2527             if self.hasOrigin:
2528                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2529             self.listExistingP4GitBranches()
2530
2531             if len(self.p4BranchesInGit) > 1:
2532                 if not self.silent:
2533                     print "Importing from/into multiple branches"
2534                 self.detectBranches = True
2535
2536             if self.verbose:
2537                 print "branches: %s" % self.p4BranchesInGit
2538
2539             p4Change = 0
2540             for branch in self.p4BranchesInGit:
2541                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2542
2543                 settings = extractSettingsGitLog(logMsg)
2544
2545                 self.readOptions(settings)
2546                 if (settings.has_key('depot-paths')
2547                     and settings.has_key ('change')):
2548                     change = int(settings['change']) + 1
2549                     p4Change = max(p4Change, change)
2550
2551                     depotPaths = sorted(settings['depot-paths'])
2552                     if self.previousDepotPaths == []:
2553                         self.previousDepotPaths = depotPaths
2554                     else:
2555                         paths = []
2556                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2557                             prev_list = prev.split("/")
2558                             cur_list = cur.split("/")
2559                             for i in range(0, min(len(cur_list), len(prev_list))):
2560                                 if cur_list[i] <> prev_list[i]:
2561                                     i = i - 1
2562                                     break
2563
2564                             paths.append ("/".join(cur_list[:i + 1]))
2565
2566                         self.previousDepotPaths = paths
2567
2568             if p4Change > 0:
2569                 self.depotPaths = sorted(self.previousDepotPaths)
2570                 self.changeRange = "@%s,#head" % p4Change
2571                 if not self.detectBranches:
2572                     self.initialParent = parseRevision(self.branch)
2573                 if not self.silent and not self.detectBranches:
2574                     print "Performing incremental import into %s git branch" % self.branch
2575
2576         if not self.branch.startswith("refs/"):
2577             self.branch = "refs/heads/" + self.branch
2578
2579         if len(args) == 0 and self.depotPaths:
2580             if not self.silent:
2581                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2582         else:
2583             if self.depotPaths and self.depotPaths != args:
2584                 print ("previous import used depot path %s and now %s was specified. "
2585                        "This doesn't work!" % (' '.join (self.depotPaths),
2586                                                ' '.join (args)))
2587                 sys.exit(1)
2588
2589             self.depotPaths = sorted(args)
2590
2591         revision = ""
2592         self.users = {}
2593
2594         # Make sure no revision specifiers are used when --changesfile
2595         # is specified.
2596         bad_changesfile = False
2597         if len(self.changesFile) > 0:
2598             for p in self.depotPaths:
2599                 if p.find("@") >= 0 or p.find("#") >= 0:
2600                     bad_changesfile = True
2601                     break
2602         if bad_changesfile:
2603             die("Option --changesfile is incompatible with revision specifiers")
2604
2605         newPaths = []
2606         for p in self.depotPaths:
2607             if p.find("@") != -1:
2608                 atIdx = p.index("@")
2609                 self.changeRange = p[atIdx:]
2610                 if self.changeRange == "@all":
2611                     self.changeRange = ""
2612                 elif ',' not in self.changeRange:
2613                     revision = self.changeRange
2614                     self.changeRange = ""
2615                 p = p[:atIdx]
2616             elif p.find("#") != -1:
2617                 hashIdx = p.index("#")
2618                 revision = p[hashIdx:]
2619                 p = p[:hashIdx]
2620             elif self.previousDepotPaths == []:
2621                 # pay attention to changesfile, if given, else import
2622                 # the entire p4 tree at the head revision
2623                 if len(self.changesFile) == 0:
2624                     revision = "#head"
2625
2626             p = re.sub ("\.\.\.$", "", p)
2627             if not p.endswith("/"):
2628                 p += "/"
2629
2630             newPaths.append(p)
2631
2632         self.depotPaths = newPaths
2633
2634         self.loadUserMapFromCache()
2635         self.labels = {}
2636         if self.detectLabels:
2637             self.getLabels();
2638
2639         if self.detectBranches:
2640             ## FIXME - what's a P4 projectName ?
2641             self.projectName = self.guessProjectName()
2642
2643             if self.hasOrigin:
2644                 self.getBranchMappingFromGitBranches()
2645             else:
2646                 self.getBranchMapping()
2647             if self.verbose:
2648                 print "p4-git branches: %s" % self.p4BranchesInGit
2649                 print "initial parents: %s" % self.initialParents
2650             for b in self.p4BranchesInGit:
2651                 if b != "master":
2652
2653                     ## FIXME
2654                     b = b[len(self.projectName):]
2655                 self.createdBranches.add(b)
2656
2657         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2658
2659         importProcess = subprocess.Popen(["git", "fast-import"],
2660                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2661                                          stderr=subprocess.PIPE);
2662         self.gitOutput = importProcess.stdout
2663         self.gitStream = importProcess.stdin
2664         self.gitError = importProcess.stderr
2665
2666         if revision:
2667             self.importHeadRevision(revision)
2668         else:
2669             changes = []
2670
2671             if len(self.changesFile) > 0:
2672                 output = open(self.changesFile).readlines()
2673                 changeSet = set()
2674                 for line in output:
2675                     changeSet.add(int(line))
2676
2677                 for change in changeSet:
2678                     changes.append(change)
2679
2680                 changes.sort()
2681             else:
2682                 # catch "git p4 sync" with no new branches, in a repo that
2683                 # does not have any existing p4 branches
2684                 if len(args) == 0 and not self.p4BranchesInGit:
2685                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2686                 if self.verbose:
2687                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2688                                                               self.changeRange)
2689                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2690
2691                 if len(self.maxChanges) > 0:
2692                     changes = changes[:min(int(self.maxChanges), len(changes))]
2693
2694             if len(changes) == 0:
2695                 if not self.silent:
2696                     print "No changes to import!"
2697             else:
2698                 if not self.silent and not self.detectBranches:
2699                     print "Import destination: %s" % self.branch
2700
2701                 self.updatedBranches = set()
2702
2703                 self.importChanges(changes)
2704
2705                 if not self.silent:
2706                     print ""
2707                     if len(self.updatedBranches) > 0:
2708                         sys.stdout.write("Updated branches: ")
2709                         for b in self.updatedBranches:
2710                             sys.stdout.write("%s " % b)
2711                         sys.stdout.write("\n")
2712
2713         if gitConfig("git-p4.importLabels", "--bool") == "true":
2714             self.importLabels = True
2715
2716         if self.importLabels:
2717             p4Labels = getP4Labels(self.depotPaths)
2718             gitTags = getGitTags()
2719
2720             missingP4Labels = p4Labels - gitTags
2721             self.importP4Labels(self.gitStream, missingP4Labels)
2722
2723         self.gitStream.close()
2724         if importProcess.wait() != 0:
2725             die("fast-import failed: %s" % self.gitError.read())
2726         self.gitOutput.close()
2727         self.gitError.close()
2728
2729         # Cleanup temporary branches created during import
2730         if self.tempBranches != []:
2731             for branch in self.tempBranches:
2732                 read_pipe("git update-ref -d %s" % branch)
2733             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
2734
2735         return True
2736
2737 class P4Rebase(Command):
2738     def __init__(self):
2739         Command.__init__(self)
2740         self.options = [
2741                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2742         ]
2743         self.importLabels = False
2744         self.description = ("Fetches the latest revision from perforce and "
2745                             + "rebases the current work (branch) against it")
2746
2747     def run(self, args):
2748         sync = P4Sync()
2749         sync.importLabels = self.importLabels
2750         sync.run([])
2751
2752         return self.rebase()
2753
2754     def rebase(self):
2755         if os.system("git update-index --refresh") != 0:
2756             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2757         if len(read_pipe("git diff-index HEAD --")) > 0:
2758             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2759
2760         [upstream, settings] = findUpstreamBranchPoint()
2761         if len(upstream) == 0:
2762             die("Cannot find upstream branchpoint for rebase")
2763
2764         # the branchpoint may be p4/foo~3, so strip off the parent
2765         upstream = re.sub("~[0-9]+$", "", upstream)
2766
2767         print "Rebasing the current branch onto %s" % upstream
2768         oldHead = read_pipe("git rev-parse HEAD").strip()
2769         system("git rebase %s" % upstream)
2770         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2771         return True
2772
2773 class P4Clone(P4Sync):
2774     def __init__(self):
2775         P4Sync.__init__(self)
2776         self.description = "Creates a new git repository and imports from Perforce into it"
2777         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2778         self.options += [
2779             optparse.make_option("--destination", dest="cloneDestination",
2780                                  action='store', default=None,
2781                                  help="where to leave result of the clone"),
2782             optparse.make_option("-/", dest="cloneExclude",
2783                                  action="append", type="string",
2784                                  help="exclude depot path"),
2785             optparse.make_option("--bare", dest="cloneBare",
2786                                  action="store_true", default=False),
2787         ]
2788         self.cloneDestination = None
2789         self.needsGit = False
2790         self.cloneBare = False
2791
2792     # This is required for the "append" cloneExclude action
2793     def ensure_value(self, attr, value):
2794         if not hasattr(self, attr) or getattr(self, attr) is None:
2795             setattr(self, attr, value)
2796         return getattr(self, attr)
2797
2798     def defaultDestination(self, args):
2799         ## TODO: use common prefix of args?
2800         depotPath = args[0]
2801         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2802         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2803         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2804         depotDir = re.sub(r"/$", "", depotDir)
2805         return os.path.split(depotDir)[1]
2806
2807     def run(self, args):
2808         if len(args) < 1:
2809             return False
2810
2811         if self.keepRepoPath and not self.cloneDestination:
2812             sys.stderr.write("Must specify destination for --keep-path\n")
2813             sys.exit(1)
2814
2815         depotPaths = args
2816
2817         if not self.cloneDestination and len(depotPaths) > 1:
2818             self.cloneDestination = depotPaths[-1]
2819             depotPaths = depotPaths[:-1]
2820
2821         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2822         for p in depotPaths:
2823             if not p.startswith("//"):
2824                 return False
2825
2826         if not self.cloneDestination:
2827             self.cloneDestination = self.defaultDestination(args)
2828
2829         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2830
2831         if not os.path.exists(self.cloneDestination):
2832             os.makedirs(self.cloneDestination)
2833         chdir(self.cloneDestination)
2834
2835         init_cmd = [ "git", "init" ]
2836         if self.cloneBare:
2837             init_cmd.append("--bare")
2838         subprocess.check_call(init_cmd)
2839
2840         if not P4Sync.run(self, depotPaths):
2841             return False
2842         if self.branch != "master":
2843             if self.importIntoRemotes:
2844                 masterbranch = "refs/remotes/p4/master"
2845             else:
2846                 masterbranch = "refs/heads/p4/master"
2847             if gitBranchExists(masterbranch):
2848                 system("git branch master %s" % masterbranch)
2849                 if not self.cloneBare:
2850                     system("git checkout -f")
2851             else:
2852                 print "Could not detect main branch. No checkout/master branch created."
2853
2854         # auto-set this variable if invoked with --use-client-spec
2855         if self.useClientSpec_from_options:
2856             system("git config --bool git-p4.useclientspec true")
2857
2858         return True
2859
2860 class P4Branches(Command):
2861     def __init__(self):
2862         Command.__init__(self)
2863         self.options = [ ]
2864         self.description = ("Shows the git branches that hold imports and their "
2865                             + "corresponding perforce depot paths")
2866         self.verbose = False
2867
2868     def run(self, args):
2869         if originP4BranchesExist():
2870             createOrUpdateBranchesFromOrigin()
2871
2872         cmdline = "git rev-parse --symbolic "
2873         cmdline += " --remotes"
2874
2875         for line in read_pipe_lines(cmdline):
2876             line = line.strip()
2877
2878             if not line.startswith('p4/') or line == "p4/HEAD":
2879                 continue
2880             branch = line
2881
2882             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2883             settings = extractSettingsGitLog(log)
2884
2885             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2886         return True
2887
2888 class HelpFormatter(optparse.IndentedHelpFormatter):
2889     def __init__(self):
2890         optparse.IndentedHelpFormatter.__init__(self)
2891
2892     def format_description(self, description):
2893         if description:
2894             return description + "\n"
2895         else:
2896             return ""
2897
2898 def printUsage(commands):
2899     print "usage: %s <command> [options]" % sys.argv[0]
2900     print ""
2901     print "valid commands: %s" % ", ".join(commands)
2902     print ""
2903     print "Try %s <command> --help for command specific help." % sys.argv[0]
2904     print ""
2905
2906 commands = {
2907     "debug" : P4Debug,
2908     "submit" : P4Submit,
2909     "commit" : P4Submit,
2910     "sync" : P4Sync,
2911     "rebase" : P4Rebase,
2912     "clone" : P4Clone,
2913     "rollback" : P4RollBack,
2914     "branches" : P4Branches
2915 }
2916
2917
2918 def main():
2919     if len(sys.argv[1:]) == 0:
2920         printUsage(commands.keys())
2921         sys.exit(2)
2922
2923     cmd = ""
2924     cmdName = sys.argv[1]
2925     try:
2926         klass = commands[cmdName]
2927         cmd = klass()
2928     except KeyError:
2929         print "unknown command %s" % cmdName
2930         print ""
2931         printUsage(commands.keys())
2932         sys.exit(2)
2933
2934     options = cmd.options
2935     cmd.gitdir = os.environ.get("GIT_DIR", None)
2936
2937     args = sys.argv[2:]
2938
2939     options.append(optparse.make_option("--verbose", dest="verbose", action="store_true"))
2940     if cmd.needsGit:
2941         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2942
2943     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2944                                    options,
2945                                    description = cmd.description,
2946                                    formatter = HelpFormatter())
2947
2948     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2949     global verbose
2950     verbose = cmd.verbose
2951     if cmd.needsGit:
2952         if cmd.gitdir == None:
2953             cmd.gitdir = os.path.abspath(".git")
2954             if not isValidGitDir(cmd.gitdir):
2955                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2956                 if os.path.exists(cmd.gitdir):
2957                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2958                     if len(cdup) > 0:
2959                         chdir(cdup);
2960
2961         if not isValidGitDir(cmd.gitdir):
2962             if isValidGitDir(cmd.gitdir + "/.git"):
2963                 cmd.gitdir += "/.git"
2964             else:
2965                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2966
2967         os.environ["GIT_DIR"] = cmd.gitdir
2968
2969     if not cmd.run(args):
2970         parser.print_help()
2971         sys.exit(2)
2972
2973
2974 if __name__ == '__main__':
2975     main()