git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re, shutil
  14
  15 verbose = False
  16
  17 # Only labels/tags matching this will be imported/exported
  18 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = ["p4"]
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += ["-u",user]
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += ["-P", password]
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += ["-p", port]
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += ["-H", host]
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += ["-c", client]
  48
  49
  50     if isinstance(cmd,basestring):
  51         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  52     else:
  53         real_cmd += cmd
  54     return real_cmd
  55
  56 def chdir(dir):
  57     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  58     # not using the shell, we have to set it ourselves.  This path could
  59     # be relative, so go there first, then figure out where we ended up.
  60     os.chdir(dir)
  61     os.environ['PWD'] = os.getcwd()
  62
  63 def die(msg):
  64     if verbose:
  65         raise Exception(msg)
  66     else:
  67         sys.stderr.write(msg + "\n")
  68         sys.exit(1)
  69
  70 def write_pipe(c, stdin):
  71     if verbose:
  72         sys.stderr.write('Writing pipe: %s\n' % str(c))
  73
  74     expand = isinstance(c,basestring)
  75     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  76     pipe = p.stdin
  77     val = pipe.write(stdin)
  78     pipe.close()
  79     if p.wait():
  80         die('Command failed: %s' % str(c))
  81
  82     return val
  83
  84 def p4_write_pipe(c, stdin):
  85     real_cmd = p4_build_cmd(c)
  86     return write_pipe(real_cmd, stdin)
  87
  88 def read_pipe(c, ignore_error=False):
  89     if verbose:
  90         sys.stderr.write('Reading pipe: %s\n' % str(c))
  91
  92     expand = isinstance(c,basestring)
  93     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  94     pipe = p.stdout
  95     val = pipe.read()
  96     if p.wait() and not ignore_error:
  97         die('Command failed: %s' % str(c))
  98
  99     return val
 100
 101 def p4_read_pipe(c, ignore_error=False):
 102     real_cmd = p4_build_cmd(c)
 103     return read_pipe(real_cmd, ignore_error)
 104
 105 def read_pipe_lines(c):
 106     if verbose:
 107         sys.stderr.write('Reading pipe: %s\n' % str(c))
 108
 109     expand = isinstance(c, basestring)
 110     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 111     pipe = p.stdout
 112     val = pipe.readlines()
 113     if pipe.close() or p.wait():
 114         die('Command failed: %s' % str(c))
 115
 116     return val
 117
 118 def p4_read_pipe_lines(c):
 119     """Specifically invoke p4 on the command supplied. """
 120     real_cmd = p4_build_cmd(c)
 121     return read_pipe_lines(real_cmd)
 122
 123 def p4_has_command(cmd):
 124     """Ask p4 for help on this command.  If it returns an error, the
 125        command does not exist in this version of p4."""
 126     real_cmd = p4_build_cmd(["help", cmd])
 127     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 128                                    stderr=subprocess.PIPE)
 129     p.communicate()
 130     return p.returncode == 0
 131
 132 def p4_has_move_command():
 133     """See if the move command exists, that it supports -k, and that
 134        it has not been administratively disabled.  The arguments
 135        must be correct, but the filenames do not have to exist.  Use
 136        ones with wildcards so even if they exist, it will fail."""
 137
 138     if not p4_has_command("move"):
 139         return False
 140     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 141     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 142     (out, err) = p.communicate()
 143     # return code will be 1 in either case
 144     if err.find("Invalid option") >= 0:
 145         return False
 146     if err.find("disabled") >= 0:
 147         return False
 148     # assume it failed because @... was invalid changelist
 149     return True
 150
 151 def system(cmd):
 152     expand = isinstance(cmd,basestring)
 153     if verbose:
 154         sys.stderr.write("executing %s\n" % str(cmd))
 155     subprocess.check_call(cmd, shell=expand)
 156
 157 def p4_system(cmd):
 158     """Specifically invoke p4 as the system command. """
 159     real_cmd = p4_build_cmd(cmd)
 160     expand = isinstance(real_cmd, basestring)
 161     subprocess.check_call(real_cmd, shell=expand)
 162
 163 def p4_integrate(src, dest):
 164     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 165
 166 def p4_sync(f, *options):
 167     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 168
 169 def p4_add(f):
 170     # forcibly add file names with wildcards
 171     if wildcard_present(f):
 172         p4_system(["add", "-f", f])
 173     else:
 174         p4_system(["add", f])
 175
 176 def p4_delete(f):
 177     p4_system(["delete", wildcard_encode(f)])
 178
 179 def p4_edit(f):
 180     p4_system(["edit", wildcard_encode(f)])
 181
 182 def p4_revert(f):
 183     p4_system(["revert", wildcard_encode(f)])
 184
 185 def p4_reopen(type, f):
 186     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 187
 188 def p4_move(src, dest):
 189     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 190
 191 def p4_describe(change):
 192     """Make sure it returns a valid result by checking for
 193        the presence of field "time".  Return a dict of the
 194        results."""
 195
 196     ds = p4CmdList(["describe", "-s", str(change)])
 197     if len(ds) != 1:
 198         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 199
 200     d = ds[0]
 201
 202     if "p4ExitCode" in d:
 203         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 204                                                       str(d)))
 205     if "code" in d:
 206         if d["code"] == "error":
 207             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 208
 209     if "time" not in d:
 210         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 211
 212     return d
 213
 214 #
 215 # Canonicalize the p4 type and return a tuple of the
 216 # base type, plus any modifiers.  See "p4 help filetypes"
 217 # for a list and explanation.
 218 #
 219 def split_p4_type(p4type):
 220
 221     p4_filetypes_historical = {
 222         "ctempobj": "binary+Sw",
 223         "ctext": "text+C",
 224         "cxtext": "text+Cx",
 225         "ktext": "text+k",
 226         "kxtext": "text+kx",
 227         "ltext": "text+F",
 228         "tempobj": "binary+FSw",
 229         "ubinary": "binary+F",
 230         "uresource": "resource+F",
 231         "uxbinary": "binary+Fx",
 232         "xbinary": "binary+x",
 233         "xltext": "text+Fx",
 234         "xtempobj": "binary+Swx",
 235         "xtext": "text+x",
 236         "xunicode": "unicode+x",
 237         "xutf16": "utf16+x",
 238     }
 239     if p4type in p4_filetypes_historical:
 240         p4type = p4_filetypes_historical[p4type]
 241     mods = ""
 242     s = p4type.split("+")
 243     base = s[0]
 244     mods = ""
 245     if len(s) > 1:
 246         mods = s[1]
 247     return (base, mods)
 248
 249 #
 250 # return the raw p4 type of a file (text, text+ko, etc)
 251 #
 252 def p4_type(file):
 253     results = p4CmdList(["fstat", "-T", "headType", file])
 254     return results[0]['headType']
 255
 256 #
 257 # Given a type base and modifier, return a regexp matching
 258 # the keywords that can be expanded in the file
 259 #
 260 def p4_keywords_regexp_for_type(base, type_mods):
 261     if base in ("text", "unicode", "binary"):
 262         kwords = None
 263         if "ko" in type_mods:
 264             kwords = 'Id|Header'
 265         elif "k" in type_mods:
 266             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 267         else:
 268             return None
 269         pattern = r"""
 270             \$              # Starts with a dollar, followed by...
 271             (%s)            # one of the keywords, followed by...
 272             (:[^$\n]+)?     # possibly an old expansion, followed by...
 273             \$              # another dollar
 274             """ % kwords
 275         return pattern
 276     else:
 277         return None
 278
 279 #
 280 # Given a file, return a regexp matching the possible
 281 # RCS keywords that will be expanded, or None for files
 282 # with kw expansion turned off.
 283 #
 284 def p4_keywords_regexp_for_file(file):
 285     if not os.path.exists(file):
 286         return None
 287     else:
 288         (type_base, type_mods) = split_p4_type(p4_type(file))
 289         return p4_keywords_regexp_for_type(type_base, type_mods)
 290
 291 def setP4ExecBit(file, mode):
 292     # Reopens an already open file and changes the execute bit to match
 293     # the execute bit setting in the passed in mode.
 294
 295     p4Type = "+x"
 296
 297     if not isModeExec(mode):
 298         p4Type = getP4OpenedType(file)
 299         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 300         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 301         if p4Type[-1] == "+":
 302             p4Type = p4Type[0:-1]
 303
 304     p4_reopen(p4Type, file)
 305
 306 def getP4OpenedType(file):
 307     # Returns the perforce file type for the given file.
 308
 309     result = p4_read_pipe(["opened", wildcard_encode(file)])
 310     match = re.match(".*\((.+)\)\r?$", result)
 311     if match:
 312         return match.group(1)
 313     else:
 314         die("Could not determine file type for %s (result: '%s')" % (file, result))
 315
 316 # Return the set of all p4 labels
 317 def getP4Labels(depotPaths):
 318     labels = set()
 319     if isinstance(depotPaths,basestring):
 320         depotPaths = [depotPaths]
 321
 322     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 323         label = l['label']
 324         labels.add(label)
 325
 326     return labels
 327
 328 # Return the set of all git tags
 329 def getGitTags():
 330     gitTags = set()
 331     for line in read_pipe_lines(["git", "tag"]):
 332         tag = line.strip()
 333         gitTags.add(tag)
 334     return gitTags
 335
 336 def diffTreePattern():
 337     # This is a simple generator for the diff tree regex pattern. This could be
 338     # a class variable if this and parseDiffTreeEntry were a part of a class.
 339     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 340     while True:
 341         yield pattern
 342
 343 def parseDiffTreeEntry(entry):
 344     """Parses a single diff tree entry into its component elements.
 345
 346     See git-diff-tree(1) manpage for details about the format of the diff
 347     output. This method returns a dictionary with the following elements:
 348
 349     src_mode - The mode of the source file
 350     dst_mode - The mode of the destination file
 351     src_sha1 - The sha1 for the source file
 352     dst_sha1 - The sha1 fr the destination file
 353     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 354     status_score - The score for the status (applicable for 'C' and 'R'
 355                    statuses). This is None if there is no score.
 356     src - The path for the source file.
 357     dst - The path for the destination file. This is only present for
 358           copy or renames. If it is not present, this is None.
 359
 360     If the pattern is not matched, None is returned."""
 361
 362     match = diffTreePattern().next().match(entry)
 363     if match:
 364         return {
 365             'src_mode': match.group(1),
 366             'dst_mode': match.group(2),
 367             'src_sha1': match.group(3),
 368             'dst_sha1': match.group(4),
 369             'status': match.group(5),
 370             'status_score': match.group(6),
 371             'src': match.group(7),
 372             'dst': match.group(10)
 373         }
 374     return None
 375
 376 def isModeExec(mode):
 377     # Returns True if the given git mode represents an executable file,
 378     # otherwise False.
 379     return mode[-3:] == "755"
 380
 381 def isModeExecChanged(src_mode, dst_mode):
 382     return isModeExec(src_mode) != isModeExec(dst_mode)
 383
 384 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 385
 386     if isinstance(cmd,basestring):
 387         cmd = "-G " + cmd
 388         expand = True
 389     else:
 390         cmd = ["-G"] + cmd
 391         expand = False
 392
 393     cmd = p4_build_cmd(cmd)
 394     if verbose:
 395         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 396
 397     # Use a temporary file to avoid deadlocks without
 398     # subprocess.communicate(), which would put another copy
 399     # of stdout into memory.
 400     stdin_file = None
 401     if stdin is not None:
 402         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 403         if isinstance(stdin,basestring):
 404             stdin_file.write(stdin)
 405         else:
 406             for i in stdin:
 407                 stdin_file.write(i + '\n')
 408         stdin_file.flush()
 409         stdin_file.seek(0)
 410
 411     p4 = subprocess.Popen(cmd,
 412                           shell=expand,
 413                           stdin=stdin_file,
 414                           stdout=subprocess.PIPE)
 415
 416     result = []
 417     try:
 418         while True:
 419             entry = marshal.load(p4.stdout)
 420             if cb is not None:
 421                 cb(entry)
 422             else:
 423                 result.append(entry)
 424     except EOFError:
 425         pass
 426     exitCode = p4.wait()
 427     if exitCode != 0:
 428         entry = {}
 429         entry["p4ExitCode"] = exitCode
 430         result.append(entry)
 431
 432     return result
 433
 434 def p4Cmd(cmd):
 435     list = p4CmdList(cmd)
 436     result = {}
 437     for entry in list:
 438         result.update(entry)
 439     return result;
 440
 441 def p4Where(depotPath):
 442     if not depotPath.endswith("/"):
 443         depotPath += "/"
 444     depotPath = depotPath + "..."
 445     outputList = p4CmdList(["where", depotPath])
 446     output = None
 447     for entry in outputList:
 448         if "depotFile" in entry:
 449             if entry["depotFile"] == depotPath:
 450                 output = entry
 451                 break
 452         elif "data" in entry:
 453             data = entry.get("data")
 454             space = data.find(" ")
 455             if data[:space] == depotPath:
 456                 output = entry
 457                 break
 458     if output == None:
 459         return ""
 460     if output["code"] == "error":
 461         return ""
 462     clientPath = ""
 463     if "path" in output:
 464         clientPath = output.get("path")
 465     elif "data" in output:
 466         data = output.get("data")
 467         lastSpace = data.rfind(" ")
 468         clientPath = data[lastSpace + 1:]
 469
 470     if clientPath.endswith("..."):
 471         clientPath = clientPath[:-3]
 472     return clientPath
 473
 474 def currentGitBranch():
 475     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 476
 477 def isValidGitDir(path):
 478     if (os.path.exists(path + "/HEAD")
 479         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 480         return True;
 481     return False
 482
 483 def parseRevision(ref):
 484     return read_pipe("git rev-parse %s" % ref).strip()
 485
 486 def branchExists(ref):
 487     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 488                      ignore_error=True)
 489     return len(rev) > 0
 490
 491 def extractLogMessageFromGitCommit(commit):
 492     logMessage = ""
 493
 494     ## fixme: title is first line of commit, not 1st paragraph.
 495     foundTitle = False
 496     for log in read_pipe_lines("git cat-file commit %s" % commit):
 497        if not foundTitle:
 498            if len(log) == 1:
 499                foundTitle = True
 500            continue
 501
 502        logMessage += log
 503     return logMessage
 504
 505 def extractSettingsGitLog(log):
 506     values = {}
 507     for line in log.split("\n"):
 508         line = line.strip()
 509         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 510         if not m:
 511             continue
 512
 513         assignments = m.group(1).split (':')
 514         for a in assignments:
 515             vals = a.split ('=')
 516             key = vals[0].strip()
 517             val = ('='.join (vals[1:])).strip()
 518             if val.endswith ('\"') and val.startswith('"'):
 519                 val = val[1:-1]
 520
 521             values[key] = val
 522
 523     paths = values.get("depot-paths")
 524     if not paths:
 525         paths = values.get("depot-path")
 526     if paths:
 527         values['depot-paths'] = paths.split(',')
 528     return values
 529
 530 def gitBranchExists(branch):
 531     proc = subprocess.Popen(["git", "rev-parse", branch],
 532                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 533     return proc.wait() == 0;
 534
 535 _gitConfig = {}
 536 def gitConfig(key, args = None): # set args to "--bool", for instance
 537     if not _gitConfig.has_key(key):
 538         argsFilter = ""
 539         if args != None:
 540             argsFilter = "%s " % args
 541         cmd = "git config %s%s" % (argsFilter, key)
 542         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 543     return _gitConfig[key]
 544
 545 def gitConfigList(key):
 546     if not _gitConfig.has_key(key):
 547         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 548     return _gitConfig[key]
 549
 550 def p4BranchesInGit(branchesAreInRemotes = True):
 551     branches = {}
 552
 553     cmdline = "git rev-parse --symbolic "
 554     if branchesAreInRemotes:
 555         cmdline += " --remotes"
 556     else:
 557         cmdline += " --branches"
 558
 559     for line in read_pipe_lines(cmdline):
 560         line = line.strip()
 561
 562         ## only import to p4/
 563         if not line.startswith('p4/') or line == "p4/HEAD":
 564             continue
 565         branch = line
 566
 567         # strip off p4
 568         branch = re.sub ("^p4/", "", line)
 569
 570         branches[branch] = parseRevision(line)
 571     return branches
 572
 573 def findUpstreamBranchPoint(head = "HEAD"):
 574     branches = p4BranchesInGit()
 575     # map from depot-path to branch name
 576     branchByDepotPath = {}
 577     for branch in branches.keys():
 578         tip = branches[branch]
 579         log = extractLogMessageFromGitCommit(tip)
 580         settings = extractSettingsGitLog(log)
 581         if settings.has_key("depot-paths"):
 582             paths = ",".join(settings["depot-paths"])
 583             branchByDepotPath[paths] = "remotes/p4/" + branch
 584
 585     settings = None
 586     parent = 0
 587     while parent < 65535:
 588         commit = head + "~%s" % parent
 589         log = extractLogMessageFromGitCommit(commit)
 590         settings = extractSettingsGitLog(log)
 591         if settings.has_key("depot-paths"):
 592             paths = ",".join(settings["depot-paths"])
 593             if branchByDepotPath.has_key(paths):
 594                 return [branchByDepotPath[paths], settings]
 595
 596         parent = parent + 1
 597
 598     return ["", settings]
 599
 600 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 601     if not silent:
 602         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 603                % localRefPrefix)
 604
 605     originPrefix = "origin/p4/"
 606
 607     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 608         line = line.strip()
 609         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 610             continue
 611
 612         headName = line[len(originPrefix):]
 613         remoteHead = localRefPrefix + headName
 614         originHead = line
 615
 616         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 617         if (not original.has_key('depot-paths')
 618             or not original.has_key('change')):
 619             continue
 620
 621         update = False
 622         if not gitBranchExists(remoteHead):
 623             if verbose:
 624                 print "creating %s" % remoteHead
 625             update = True
 626         else:
 627             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 628             if settings.has_key('change') > 0:
 629                 if settings['depot-paths'] == original['depot-paths']:
 630                     originP4Change = int(original['change'])
 631                     p4Change = int(settings['change'])
 632                     if originP4Change > p4Change:
 633                         print ("%s (%s) is newer than %s (%s). "
 634                                "Updating p4 branch from origin."
 635                                % (originHead, originP4Change,
 636                                   remoteHead, p4Change))
 637                         update = True
 638                 else:
 639                     print ("Ignoring: %s was imported from %s while "
 640                            "%s was imported from %s"
 641                            % (originHead, ','.join(original['depot-paths']),
 642                               remoteHead, ','.join(settings['depot-paths'])))
 643
 644         if update:
 645             system("git update-ref %s %s" % (remoteHead, originHead))
 646
 647 def originP4BranchesExist():
 648         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 649
 650 def p4ChangesForPaths(depotPaths, changeRange):
 651     assert depotPaths
 652     cmd = ['changes']
 653     for p in depotPaths:
 654         cmd += ["%s...%s" % (p, changeRange)]
 655     output = p4_read_pipe_lines(cmd)
 656
 657     changes = {}
 658     for line in output:
 659         changeNum = int(line.split(" ")[1])
 660         changes[changeNum] = True
 661
 662     changelist = changes.keys()
 663     changelist.sort()
 664     return changelist
 665
 666 def p4PathStartsWith(path, prefix):
 667     # This method tries to remedy a potential mixed-case issue:
 668     #
 669     # If UserA adds  //depot/DirA/file1
 670     # and UserB adds //depot/dira/file2
 671     #
 672     # we may or may not have a problem. If you have core.ignorecase=true,
 673     # we treat DirA and dira as the same directory
 674     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 675     if ignorecase:
 676         return path.lower().startswith(prefix.lower())
 677     return path.startswith(prefix)
 678
 679 def getClientSpec():
 680     """Look at the p4 client spec, create a View() object that contains
 681        all the mappings, and return it."""
 682
 683     specList = p4CmdList("client -o")
 684     if len(specList) != 1:
 685         die('Output from "client -o" is %d lines, expecting 1' %
 686             len(specList))
 687
 688     # dictionary of all client parameters
 689     entry = specList[0]
 690
 691     # just the keys that start with "View"
 692     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 693
 694     # hold this new View
 695     view = View()
 696
 697     # append the lines, in order, to the view
 698     for view_num in range(len(view_keys)):
 699         k = "View%d" % view_num
 700         if k not in view_keys:
 701             die("Expected view key %s missing" % k)
 702         view.append(entry[k])
 703
 704     return view
 705
 706 def getClientRoot():
 707     """Grab the client directory."""
 708
 709     output = p4CmdList("client -o")
 710     if len(output) != 1:
 711         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 712
 713     entry = output[0]
 714     if "Root" not in entry:
 715         die('Client has no "Root"')
 716
 717     return entry["Root"]
 718
 719 #
 720 # P4 wildcards are not allowed in filenames.  P4 complains
 721 # if you simply add them, but you can force it with "-f", in
 722 # which case it translates them into %xx encoding internally.
 723 #
 724 def wildcard_decode(path):
 725     # Search for and fix just these four characters.  Do % last so
 726     # that fixing it does not inadvertently create new %-escapes.
 727     # Cannot have * in a filename in windows; untested as to
 728     # what p4 would do in such a case.
 729     if not platform.system() == "Windows":
 730         path = path.replace("%2A", "*")
 731     path = path.replace("%23", "#") \
 732                .replace("%40", "@") \
 733                .replace("%25", "%")
 734     return path
 735
 736 def wildcard_encode(path):
 737     # do % first to avoid double-encoding the %s introduced here
 738     path = path.replace("%", "%25") \
 739                .replace("*", "%2A") \
 740                .replace("#", "%23") \
 741                .replace("@", "%40")
 742     return path
 743
 744 def wildcard_present(path):
 745     return path.translate(None, "*#@%") != path
 746
 747 class Command:
 748     def __init__(self):
 749         self.usage = "usage: %prog [options]"
 750         self.needsGit = True
 751         self.verbose = False
 752
 753 class P4UserMap:
 754     def __init__(self):
 755         self.userMapFromPerforceServer = False
 756         self.myP4UserId = None
 757
 758     def p4UserId(self):
 759         if self.myP4UserId:
 760             return self.myP4UserId
 761
 762         results = p4CmdList("user -o")
 763         for r in results:
 764             if r.has_key('User'):
 765                 self.myP4UserId = r['User']
 766                 return r['User']
 767         die("Could not find your p4 user id")
 768
 769     def p4UserIsMe(self, p4User):
 770         # return True if the given p4 user is actually me
 771         me = self.p4UserId()
 772         if not p4User or p4User != me:
 773             return False
 774         else:
 775             return True
 776
 777     def getUserCacheFilename(self):
 778         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 779         return home + "/.gitp4-usercache.txt"
 780
 781     def getUserMapFromPerforceServer(self):
 782         if self.userMapFromPerforceServer:
 783             return
 784         self.users = {}
 785         self.emails = {}
 786
 787         for output in p4CmdList("users"):
 788             if not output.has_key("User"):
 789                 continue
 790             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 791             self.emails[output["Email"]] = output["User"]
 792
 793
 794         s = ''
 795         for (key, val) in self.users.items():
 796             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 797
 798         open(self.getUserCacheFilename(), "wb").write(s)
 799         self.userMapFromPerforceServer = True
 800
 801     def loadUserMapFromCache(self):
 802         self.users = {}
 803         self.userMapFromPerforceServer = False
 804         try:
 805             cache = open(self.getUserCacheFilename(), "rb")
 806             lines = cache.readlines()
 807             cache.close()
 808             for line in lines:
 809                 entry = line.strip().split("\t")
 810                 self.users[entry[0]] = entry[1]
 811         except IOError:
 812             self.getUserMapFromPerforceServer()
 813
 814 class P4Debug(Command):
 815     def __init__(self):
 816         Command.__init__(self)
 817         self.options = []
 818         self.description = "A tool to debug the output of p4 -G."
 819         self.needsGit = False
 820
 821     def run(self, args):
 822         j = 0
 823         for output in p4CmdList(args):
 824             print 'Element: %d' % j
 825             j += 1
 826             print output
 827         return True
 828
 829 class P4RollBack(Command):
 830     def __init__(self):
 831         Command.__init__(self)
 832         self.options = [
 833             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 834         ]
 835         self.description = "A tool to debug the multi-branch import. Don't use :)"
 836         self.rollbackLocalBranches = False
 837
 838     def run(self, args):
 839         if len(args) != 1:
 840             return False
 841         maxChange = int(args[0])
 842
 843         if "p4ExitCode" in p4Cmd("changes -m 1"):
 844             die("Problems executing p4");
 845
 846         if self.rollbackLocalBranches:
 847             refPrefix = "refs/heads/"
 848             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 849         else:
 850             refPrefix = "refs/remotes/"
 851             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 852
 853         for line in lines:
 854             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 855                 line = line.strip()
 856                 ref = refPrefix + line
 857                 log = extractLogMessageFromGitCommit(ref)
 858                 settings = extractSettingsGitLog(log)
 859
 860                 depotPaths = settings['depot-paths']
 861                 change = settings['change']
 862
 863                 changed = False
 864
 865                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 866                                                            for p in depotPaths]))) == 0:
 867                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 868                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 869                     continue
 870
 871                 while change and int(change) > maxChange:
 872                     changed = True
 873                     if self.verbose:
 874                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 875                     system("git update-ref %s \"%s^\"" % (ref, ref))
 876                     log = extractLogMessageFromGitCommit(ref)
 877                     settings =  extractSettingsGitLog(log)
 878
 879
 880                     depotPaths = settings['depot-paths']
 881                     change = settings['change']
 882
 883                 if changed:
 884                     print "%s rewound to %s" % (ref, change)
 885
 886         return True
 887
 888 class P4Submit(Command, P4UserMap):
 889
 890     conflict_behavior_choices = ("ask", "skip", "quit")
 891
 892     def __init__(self):
 893         Command.__init__(self)
 894         P4UserMap.__init__(self)
 895         self.options = [
 896                 optparse.make_option("--origin", dest="origin"),
 897                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 898                 # preserve the user, requires relevant p4 permissions
 899                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 900                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 901                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
 902                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
 903                 optparse.make_option("--conflict", dest="conflict_behavior",
 904                                      choices=self.conflict_behavior_choices)
 905         ]
 906         self.description = "Submit changes from git to the perforce depot."
 907         self.usage += " [name of git branch to submit into perforce depot]"
 908         self.origin = ""
 909         self.detectRenames = False
 910         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 911         self.dry_run = False
 912         self.prepare_p4_only = False
 913         self.conflict_behavior = None
 914         self.isWindows = (platform.system() == "Windows")
 915         self.exportLabels = False
 916         self.p4HasMoveCommand = p4_has_move_command()
 917
 918     def check(self):
 919         if len(p4CmdList("opened ...")) > 0:
 920             die("You have files opened with perforce! Close them before starting the sync.")
 921
 922     def separate_jobs_from_description(self, message):
 923         """Extract and return a possible Jobs field in the commit
 924            message.  It goes into a separate section in the p4 change
 925            specification.
 926
 927            A jobs line starts with "Jobs:" and looks like a new field
 928            in a form.  Values are white-space separated on the same
 929            line or on following lines that start with a tab.
 930
 931            This does not parse and extract the full git commit message
 932            like a p4 form.  It just sees the Jobs: line as a marker
 933            to pass everything from then on directly into the p4 form,
 934            but outside the description section.
 935
 936            Return a tuple (stripped log message, jobs string)."""
 937
 938         m = re.search(r'^Jobs:', message, re.MULTILINE)
 939         if m is None:
 940             return (message, None)
 941
 942         jobtext = message[m.start():]
 943         stripped_message = message[:m.start()].rstrip()
 944         return (stripped_message, jobtext)
 945
 946     def prepareLogMessage(self, template, message, jobs):
 947         """Edits the template returned from "p4 change -o" to insert
 948            the message in the Description field, and the jobs text in
 949            the Jobs field."""
 950         result = ""
 951
 952         inDescriptionSection = False
 953
 954         for line in template.split("\n"):
 955             if line.startswith("#"):
 956                 result += line + "\n"
 957                 continue
 958
 959             if inDescriptionSection:
 960                 if line.startswith("Files:") or line.startswith("Jobs:"):
 961                     inDescriptionSection = False
 962                     # insert Jobs section
 963                     if jobs:
 964                         result += jobs + "\n"
 965                 else:
 966                     continue
 967             else:
 968                 if line.startswith("Description:"):
 969                     inDescriptionSection = True
 970                     line += "\n"
 971                     for messageLine in message.split("\n"):
 972                         line += "\t" + messageLine + "\n"
 973
 974             result += line + "\n"
 975
 976         return result
 977
 978     def patchRCSKeywords(self, file, pattern):
 979         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
 980         (handle, outFileName) = tempfile.mkstemp(dir='.')
 981         try:
 982             outFile = os.fdopen(handle, "w+")
 983             inFile = open(file, "r")
 984             regexp = re.compile(pattern, re.VERBOSE)
 985             for line in inFile.readlines():
 986                 line = regexp.sub(r'$\1$', line)
 987                 outFile.write(line)
 988             inFile.close()
 989             outFile.close()
 990             # Forcibly overwrite the original file
 991             os.unlink(file)
 992             shutil.move(outFileName, file)
 993         except:
 994             # cleanup our temporary file
 995             os.unlink(outFileName)
 996             print "Failed to strip RCS keywords in %s" % file
 997             raise
 998
 999         print "Patched up RCS keywords in %s" % file
1000
1001     def p4UserForCommit(self,id):
1002         # Return the tuple (perforce user,git email) for a given git commit id
1003         self.getUserMapFromPerforceServer()
1004         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
1005         gitEmail = gitEmail.strip()
1006         if not self.emails.has_key(gitEmail):
1007             return (None,gitEmail)
1008         else:
1009             return (self.emails[gitEmail],gitEmail)
1010
1011     def checkValidP4Users(self,commits):
1012         # check if any git authors cannot be mapped to p4 users
1013         for id in commits:
1014             (user,email) = self.p4UserForCommit(id)
1015             if not user:
1016                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1017                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
1018                     print "%s" % msg
1019                 else:
1020                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1021
1022     def lastP4Changelist(self):
1023         # Get back the last changelist number submitted in this client spec. This
1024         # then gets used to patch up the username in the change. If the same
1025         # client spec is being used by multiple processes then this might go
1026         # wrong.
1027         results = p4CmdList("client -o")        # find the current client
1028         client = None
1029         for r in results:
1030             if r.has_key('Client'):
1031                 client = r['Client']
1032                 break
1033         if not client:
1034             die("could not get client spec")
1035         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1036         for r in results:
1037             if r.has_key('change'):
1038                 return r['change']
1039         die("Could not get changelist number for last submit - cannot patch up user details")
1040
1041     def modifyChangelistUser(self, changelist, newUser):
1042         # fixup the user field of a changelist after it has been submitted.
1043         changes = p4CmdList("change -o %s" % changelist)
1044         if len(changes) != 1:
1045             die("Bad output from p4 change modifying %s to user %s" %
1046                 (changelist, newUser))
1047
1048         c = changes[0]
1049         if c['User'] == newUser: return   # nothing to do
1050         c['User'] = newUser
1051         input = marshal.dumps(c)
1052
1053         result = p4CmdList("change -f -i", stdin=input)
1054         for r in result:
1055             if r.has_key('code'):
1056                 if r['code'] == 'error':
1057                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1058             if r.has_key('data'):
1059                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1060                 return
1061         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1062
1063     def canChangeChangelists(self):
1064         # check to see if we have p4 admin or super-user permissions, either of
1065         # which are required to modify changelists.
1066         results = p4CmdList(["protects", self.depotPath])
1067         for r in results:
1068             if r.has_key('perm'):
1069                 if r['perm'] == 'admin':
1070                     return 1
1071                 if r['perm'] == 'super':
1072                     return 1
1073         return 0
1074
1075     def prepareSubmitTemplate(self):
1076         """Run "p4 change -o" to grab a change specification template.
1077            This does not use "p4 -G", as it is nice to keep the submission
1078            template in original order, since a human might edit it.
1079
1080            Remove lines in the Files section that show changes to files
1081            outside the depot path we're committing into."""
1082
1083         template = ""
1084         inFilesSection = False
1085         for line in p4_read_pipe_lines(['change', '-o']):
1086             if line.endswith("\r\n"):
1087                 line = line[:-2] + "\n"
1088             if inFilesSection:
1089                 if line.startswith("\t"):
1090                     # path starts and ends with a tab
1091                     path = line[1:]
1092                     lastTab = path.rfind("\t")
1093                     if lastTab != -1:
1094                         path = path[:lastTab]
1095                         if not p4PathStartsWith(path, self.depotPath):
1096                             continue
1097                 else:
1098                     inFilesSection = False
1099             else:
1100                 if line.startswith("Files:"):
1101                     inFilesSection = True
1102
1103             template += line
1104
1105         return template
1106
1107     def edit_template(self, template_file):
1108         """Invoke the editor to let the user change the submission
1109            message.  Return true if okay to continue with the submit."""
1110
1111         # if configured to skip the editing part, just submit
1112         if gitConfig("git-p4.skipSubmitEdit") == "true":
1113             return True
1114
1115         # look at the modification time, to check later if the user saved
1116         # the file
1117         mtime = os.stat(template_file).st_mtime
1118
1119         # invoke the editor
1120         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1121             editor = os.environ.get("P4EDITOR")
1122         else:
1123             editor = read_pipe("git var GIT_EDITOR").strip()
1124         system(editor + " " + template_file)
1125
1126         # If the file was not saved, prompt to see if this patch should
1127         # be skipped.  But skip this verification step if configured so.
1128         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1129             return True
1130
1131         # modification time updated means user saved the file
1132         if os.stat(template_file).st_mtime > mtime:
1133             return True
1134
1135         while True:
1136             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1137             if response == 'y':
1138                 return True
1139             if response == 'n':
1140                 return False
1141
1142     def applyCommit(self, id):
1143         """Apply one commit, return True if it succeeded."""
1144
1145         print "Applying", read_pipe(["git", "show", "-s",
1146                                      "--format=format:%h %s", id])
1147
1148         (p4User, gitEmail) = self.p4UserForCommit(id)
1149
1150         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1151         filesToAdd = set()
1152         filesToDelete = set()
1153         editedFiles = set()
1154         pureRenameCopy = set()
1155         filesToChangeExecBit = {}
1156
1157         for line in diff:
1158             diff = parseDiffTreeEntry(line)
1159             modifier = diff['status']
1160             path = diff['src']
1161             if modifier == "M":
1162                 p4_edit(path)
1163                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1164                     filesToChangeExecBit[path] = diff['dst_mode']
1165                 editedFiles.add(path)
1166             elif modifier == "A":
1167                 filesToAdd.add(path)
1168                 filesToChangeExecBit[path] = diff['dst_mode']
1169                 if path in filesToDelete:
1170                     filesToDelete.remove(path)
1171             elif modifier == "D":
1172                 filesToDelete.add(path)
1173                 if path in filesToAdd:
1174                     filesToAdd.remove(path)
1175             elif modifier == "C":
1176                 src, dest = diff['src'], diff['dst']
1177                 p4_integrate(src, dest)
1178                 pureRenameCopy.add(dest)
1179                 if diff['src_sha1'] != diff['dst_sha1']:
1180                     p4_edit(dest)
1181                     pureRenameCopy.discard(dest)
1182                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1183                     p4_edit(dest)
1184                     pureRenameCopy.discard(dest)
1185                     filesToChangeExecBit[dest] = diff['dst_mode']
1186                 os.unlink(dest)
1187                 editedFiles.add(dest)
1188             elif modifier == "R":
1189                 src, dest = diff['src'], diff['dst']
1190                 if self.p4HasMoveCommand:
1191                     p4_edit(src)        # src must be open before move
1192                     p4_move(src, dest)  # opens for (move/delete, move/add)
1193                 else:
1194                     p4_integrate(src, dest)
1195                     if diff['src_sha1'] != diff['dst_sha1']:
1196                         p4_edit(dest)
1197                     else:
1198                         pureRenameCopy.add(dest)
1199                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1200                     if not self.p4HasMoveCommand:
1201                         p4_edit(dest)   # with move: already open, writable
1202                     filesToChangeExecBit[dest] = diff['dst_mode']
1203                 if not self.p4HasMoveCommand:
1204                     os.unlink(dest)
1205                     filesToDelete.add(src)
1206                 editedFiles.add(dest)
1207             else:
1208                 die("unknown modifier %s for %s" % (modifier, path))
1209
1210         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1211         patchcmd = diffcmd + " | git apply "
1212         tryPatchCmd = patchcmd + "--check -"
1213         applyPatchCmd = patchcmd + "--check --apply -"
1214         patch_succeeded = True
1215
1216         if os.system(tryPatchCmd) != 0:
1217             fixed_rcs_keywords = False
1218             patch_succeeded = False
1219             print "Unfortunately applying the change failed!"
1220
1221             # Patch failed, maybe it's just RCS keyword woes. Look through
1222             # the patch to see if that's possible.
1223             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1224                 file = None
1225                 pattern = None
1226                 kwfiles = {}
1227                 for file in editedFiles | filesToDelete:
1228                     # did this file's delta contain RCS keywords?
1229                     pattern = p4_keywords_regexp_for_file(file)
1230
1231                     if pattern:
1232                         # this file is a possibility...look for RCS keywords.
1233                         regexp = re.compile(pattern, re.VERBOSE)
1234                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1235                             if regexp.search(line):
1236                                 if verbose:
1237                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1238                                 kwfiles[file] = pattern
1239                                 break
1240
1241                 for file in kwfiles:
1242                     if verbose:
1243                         print "zapping %s with %s" % (line,pattern)
1244                     self.patchRCSKeywords(file, kwfiles[file])
1245                     fixed_rcs_keywords = True
1246
1247             if fixed_rcs_keywords:
1248                 print "Retrying the patch with RCS keywords cleaned up"
1249                 if os.system(tryPatchCmd) == 0:
1250                     patch_succeeded = True
1251
1252         if not patch_succeeded:
1253             for f in editedFiles:
1254                 p4_revert(f)
1255             return False
1256
1257         #
1258         # Apply the patch for real, and do add/delete/+x handling.
1259         #
1260         system(applyPatchCmd)
1261
1262         for f in filesToAdd:
1263             p4_add(f)
1264         for f in filesToDelete:
1265             p4_revert(f)
1266             p4_delete(f)
1267
1268         # Set/clear executable bits
1269         for f in filesToChangeExecBit.keys():
1270             mode = filesToChangeExecBit[f]
1271             setP4ExecBit(f, mode)
1272
1273         #
1274         # Build p4 change description, starting with the contents
1275         # of the git commit message.
1276         #
1277         logMessage = extractLogMessageFromGitCommit(id)
1278         logMessage = logMessage.strip()
1279         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1280
1281         template = self.prepareSubmitTemplate()
1282         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1283
1284         if self.preserveUser:
1285            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1286
1287         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1288             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1289             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1290             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1291
1292         separatorLine = "######## everything below this line is just the diff #######\n"
1293
1294         # diff
1295         if os.environ.has_key("P4DIFF"):
1296             del(os.environ["P4DIFF"])
1297         diff = ""
1298         for editedFile in editedFiles:
1299             diff += p4_read_pipe(['diff', '-du',
1300                                   wildcard_encode(editedFile)])
1301
1302         # new file diff
1303         newdiff = ""
1304         for newFile in filesToAdd:
1305             newdiff += "==== new file ====\n"
1306             newdiff += "--- /dev/null\n"
1307             newdiff += "+++ %s\n" % newFile
1308             f = open(newFile, "r")
1309             for line in f.readlines():
1310                 newdiff += "+" + line
1311             f.close()
1312
1313         # change description file: submitTemplate, separatorLine, diff, newdiff
1314         (handle, fileName) = tempfile.mkstemp()
1315         tmpFile = os.fdopen(handle, "w+")
1316         if self.isWindows:
1317             submitTemplate = submitTemplate.replace("\n", "\r\n")
1318             separatorLine = separatorLine.replace("\n", "\r\n")
1319             newdiff = newdiff.replace("\n", "\r\n")
1320         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1321         tmpFile.close()
1322
1323         if self.prepare_p4_only:
1324             #
1325             # Leave the p4 tree prepared, and the submit template around
1326             # and let the user decide what to do next
1327             #
1328             print
1329             print "P4 workspace prepared for submission."
1330             print "To submit or revert, go to client workspace"
1331             print "  " + self.clientPath
1332             print
1333             print "To submit, use \"p4 submit\" to write a new description,"
1334             print "or \"p4 submit -i %s\" to use the one prepared by" \
1335                   " \"git p4\"." % fileName
1336             print "You can delete the file \"%s\" when finished." % fileName
1337
1338             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1339                 print "To preserve change ownership by user %s, you must\n" \
1340                       "do \"p4 change -f <change>\" after submitting and\n" \
1341                       "edit the User field."
1342             if pureRenameCopy:
1343                 print "After submitting, renamed files must be re-synced."
1344                 print "Invoke \"p4 sync -f\" on each of these files:"
1345                 for f in pureRenameCopy:
1346                     print "  " + f
1347
1348             print
1349             print "To revert the changes, use \"p4 revert ...\", and delete"
1350             print "the submit template file \"%s\"" % fileName
1351             if filesToAdd:
1352                 print "Since the commit adds new files, they must be deleted:"
1353                 for f in filesToAdd:
1354                     print "  " + f
1355             print
1356             return True
1357
1358         #
1359         # Let the user edit the change description, then submit it.
1360         #
1361         if self.edit_template(fileName):
1362             # read the edited message and submit
1363             ret = True
1364             tmpFile = open(fileName, "rb")
1365             message = tmpFile.read()
1366             tmpFile.close()
1367             submitTemplate = message[:message.index(separatorLine)]
1368             if self.isWindows:
1369                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1370             p4_write_pipe(['submit', '-i'], submitTemplate)
1371
1372             if self.preserveUser:
1373                 if p4User:
1374                     # Get last changelist number. Cannot easily get it from
1375                     # the submit command output as the output is
1376                     # unmarshalled.
1377                     changelist = self.lastP4Changelist()
1378                     self.modifyChangelistUser(changelist, p4User)
1379
1380             # The rename/copy happened by applying a patch that created a
1381             # new file.  This leaves it writable, which confuses p4.
1382             for f in pureRenameCopy:
1383                 p4_sync(f, "-f")
1384
1385         else:
1386             # skip this patch
1387             ret = False
1388             print "Submission cancelled, undoing p4 changes."
1389             for f in editedFiles:
1390                 p4_revert(f)
1391             for f in filesToAdd:
1392                 p4_revert(f)
1393                 os.remove(f)
1394             for f in filesToDelete:
1395                 p4_revert(f)
1396
1397         os.remove(fileName)
1398         return ret
1399
1400     # Export git tags as p4 labels. Create a p4 label and then tag
1401     # with that.
1402     def exportGitTags(self, gitTags):
1403         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1404         if len(validLabelRegexp) == 0:
1405             validLabelRegexp = defaultLabelRegexp
1406         m = re.compile(validLabelRegexp)
1407
1408         for name in gitTags:
1409
1410             if not m.match(name):
1411                 if verbose:
1412                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1413                 continue
1414
1415             # Get the p4 commit this corresponds to
1416             logMessage = extractLogMessageFromGitCommit(name)
1417             values = extractSettingsGitLog(logMessage)
1418
1419             if not values.has_key('change'):
1420                 # a tag pointing to something not sent to p4; ignore
1421                 if verbose:
1422                     print "git tag %s does not give a p4 commit" % name
1423                 continue
1424             else:
1425                 changelist = values['change']
1426
1427             # Get the tag details.
1428             inHeader = True
1429             isAnnotated = False
1430             body = []
1431             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1432                 l = l.strip()
1433                 if inHeader:
1434                     if re.match(r'tag\s+', l):
1435                         isAnnotated = True
1436                     elif re.match(r'\s*$', l):
1437                         inHeader = False
1438                         continue
1439                 else:
1440                     body.append(l)
1441
1442             if not isAnnotated:
1443                 body = ["lightweight tag imported by git p4\n"]
1444
1445             # Create the label - use the same view as the client spec we are using
1446             clientSpec = getClientSpec()
1447
1448             labelTemplate  = "Label: %s\n" % name
1449             labelTemplate += "Description:\n"
1450             for b in body:
1451                 labelTemplate += "\t" + b + "\n"
1452             labelTemplate += "View:\n"
1453             for mapping in clientSpec.mappings:
1454                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1455
1456             if self.dry_run:
1457                 print "Would create p4 label %s for tag" % name
1458             elif self.prepare_p4_only:
1459                 print "Not creating p4 label %s for tag due to option" \
1460                       " --prepare-p4-only" % name
1461             else:
1462                 p4_write_pipe(["label", "-i"], labelTemplate)
1463
1464                 # Use the label
1465                 p4_system(["tag", "-l", name] +
1466                           ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1467
1468                 if verbose:
1469                     print "created p4 label for tag %s" % name
1470
1471     def run(self, args):
1472         if len(args) == 0:
1473             self.master = currentGitBranch()
1474             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1475                 die("Detecting current git branch failed!")
1476         elif len(args) == 1:
1477             self.master = args[0]
1478             if not branchExists(self.master):
1479                 die("Branch %s does not exist" % self.master)
1480         else:
1481             return False
1482
1483         allowSubmit = gitConfig("git-p4.allowSubmit")
1484         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1485             die("%s is not in git-p4.allowSubmit" % self.master)
1486
1487         [upstream, settings] = findUpstreamBranchPoint()
1488         self.depotPath = settings['depot-paths'][0]
1489         if len(self.origin) == 0:
1490             self.origin = upstream
1491
1492         if self.preserveUser:
1493             if not self.canChangeChangelists():
1494                 die("Cannot preserve user names without p4 super-user or admin permissions")
1495
1496         # if not set from the command line, try the config file
1497         if self.conflict_behavior is None:
1498             val = gitConfig("git-p4.conflict")
1499             if val:
1500                 if val not in self.conflict_behavior_choices:
1501                     die("Invalid value '%s' for config git-p4.conflict" % val)
1502             else:
1503                 val = "ask"
1504             self.conflict_behavior = val
1505
1506         if self.verbose:
1507             print "Origin branch is " + self.origin
1508
1509         if len(self.depotPath) == 0:
1510             print "Internal error: cannot locate perforce depot path from existing branches"
1511             sys.exit(128)
1512
1513         self.useClientSpec = False
1514         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1515             self.useClientSpec = True
1516         if self.useClientSpec:
1517             self.clientSpecDirs = getClientSpec()
1518
1519         if self.useClientSpec:
1520             # all files are relative to the client spec
1521             self.clientPath = getClientRoot()
1522         else:
1523             self.clientPath = p4Where(self.depotPath)
1524
1525         if self.clientPath == "":
1526             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1527
1528         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1529         self.oldWorkingDirectory = os.getcwd()
1530
1531         # ensure the clientPath exists
1532         new_client_dir = False
1533         if not os.path.exists(self.clientPath):
1534             new_client_dir = True
1535             os.makedirs(self.clientPath)
1536
1537         chdir(self.clientPath)
1538         if self.dry_run:
1539             print "Would synchronize p4 checkout in %s" % self.clientPath
1540         else:
1541             print "Synchronizing p4 checkout..."
1542             if new_client_dir:
1543                 # old one was destroyed, and maybe nobody told p4
1544                 p4_sync("...", "-f")
1545             else:
1546                 p4_sync("...")
1547         self.check()
1548
1549         commits = []
1550         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1551             commits.append(line.strip())
1552         commits.reverse()
1553
1554         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1555             self.checkAuthorship = False
1556         else:
1557             self.checkAuthorship = True
1558
1559         if self.preserveUser:
1560             self.checkValidP4Users(commits)
1561
1562         #
1563         # Build up a set of options to be passed to diff when
1564         # submitting each commit to p4.
1565         #
1566         if self.detectRenames:
1567             # command-line -M arg
1568             self.diffOpts = "-M"
1569         else:
1570             # If not explicitly set check the config variable
1571             detectRenames = gitConfig("git-p4.detectRenames")
1572
1573             if detectRenames.lower() == "false" or detectRenames == "":
1574                 self.diffOpts = ""
1575             elif detectRenames.lower() == "true":
1576                 self.diffOpts = "-M"
1577             else:
1578                 self.diffOpts = "-M%s" % detectRenames
1579
1580         # no command-line arg for -C or --find-copies-harder, just
1581         # config variables
1582         detectCopies = gitConfig("git-p4.detectCopies")
1583         if detectCopies.lower() == "false" or detectCopies == "":
1584             pass
1585         elif detectCopies.lower() == "true":
1586             self.diffOpts += " -C"
1587         else:
1588             self.diffOpts += " -C%s" % detectCopies
1589
1590         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1591             self.diffOpts += " --find-copies-harder"
1592
1593         #
1594         # Apply the commits, one at a time.  On failure, ask if should
1595         # continue to try the rest of the patches, or quit.
1596         #
1597         if self.dry_run:
1598             print "Would apply"
1599         applied = []
1600         last = len(commits) - 1
1601         for i, commit in enumerate(commits):
1602             if self.dry_run:
1603                 print " ", read_pipe(["git", "show", "-s",
1604                                       "--format=format:%h %s", commit])
1605                 ok = True
1606             else:
1607                 ok = self.applyCommit(commit)
1608             if ok:
1609                 applied.append(commit)
1610             else:
1611                 if self.prepare_p4_only and i < last:
1612                     print "Processing only the first commit due to option" \
1613                           " --prepare-p4-only"
1614                     break
1615                 if i < last:
1616                     quit = False
1617                     while True:
1618                         # prompt for what to do, or use the option/variable
1619                         if self.conflict_behavior == "ask":
1620                             print "What do you want to do?"
1621                             response = raw_input("[s]kip this commit but apply"
1622                                                  " the rest, or [q]uit? ")
1623                             if not response:
1624                                 continue
1625                         elif self.conflict_behavior == "skip":
1626                             response = "s"
1627                         elif self.conflict_behavior == "quit":
1628                             response = "q"
1629                         else:
1630                             die("Unknown conflict_behavior '%s'" %
1631                                 self.conflict_behavior)
1632
1633                         if response[0] == "s":
1634                             print "Skipping this commit, but applying the rest"
1635                             break
1636                         if response[0] == "q":
1637                             print "Quitting"
1638                             quit = True
1639                             break
1640                     if quit:
1641                         break
1642
1643         chdir(self.oldWorkingDirectory)
1644
1645         if self.dry_run:
1646             pass
1647         elif self.prepare_p4_only:
1648             pass
1649         elif len(commits) == len(applied):
1650             print "All commits applied!"
1651
1652             sync = P4Sync()
1653             sync.run([])
1654
1655             rebase = P4Rebase()
1656             rebase.rebase()
1657
1658         else:
1659             if len(applied) == 0:
1660                 print "No commits applied."
1661             else:
1662                 print "Applied only the commits marked with '*':"
1663                 for c in commits:
1664                     if c in applied:
1665                         star = "*"
1666                     else:
1667                         star = " "
1668                     print star, read_pipe(["git", "show", "-s",
1669                                            "--format=format:%h %s",  c])
1670                 print "You will have to do 'git p4 sync' and rebase."
1671
1672         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1673             self.exportLabels = True
1674
1675         if self.exportLabels:
1676             p4Labels = getP4Labels(self.depotPath)
1677             gitTags = getGitTags()
1678
1679             missingGitTags = gitTags - p4Labels
1680             self.exportGitTags(missingGitTags)
1681
1682         # exit with error unless everything applied perfecly
1683         if len(commits) != len(applied):
1684                 sys.exit(1)
1685
1686         return True
1687
1688 class View(object):
1689     """Represent a p4 view ("p4 help views"), and map files in a
1690        repo according to the view."""
1691
1692     class Path(object):
1693         """A depot or client path, possibly containing wildcards.
1694            The only one supported is ... at the end, currently.
1695            Initialize with the full path, with //depot or //client."""
1696
1697         def __init__(self, path, is_depot):
1698             self.path = path
1699             self.is_depot = is_depot
1700             self.find_wildcards()
1701             # remember the prefix bit, useful for relative mappings
1702             m = re.match("(//[^/]+/)", self.path)
1703             if not m:
1704                 die("Path %s does not start with //prefix/" % self.path)
1705             prefix = m.group(1)
1706             if not self.is_depot:
1707                 # strip //client/ on client paths
1708                 self.path = self.path[len(prefix):]
1709
1710         def find_wildcards(self):
1711             """Make sure wildcards are valid, and set up internal
1712                variables."""
1713
1714             self.ends_triple_dot = False
1715             # There are three wildcards allowed in p4 views
1716             # (see "p4 help views").  This code knows how to
1717             # handle "..." (only at the end), but cannot deal with
1718             # "%%n" or "*".  Only check the depot_side, as p4 should
1719             # validate that the client_side matches too.
1720             if re.search(r'%%[1-9]', self.path):
1721                 die("Can't handle %%n wildcards in view: %s" % self.path)
1722             if self.path.find("*") >= 0:
1723                 die("Can't handle * wildcards in view: %s" % self.path)
1724             triple_dot_index = self.path.find("...")
1725             if triple_dot_index >= 0:
1726                 if triple_dot_index != len(self.path) - 3:
1727                     die("Can handle only single ... wildcard, at end: %s" %
1728                         self.path)
1729                 self.ends_triple_dot = True
1730
1731         def ensure_compatible(self, other_path):
1732             """Make sure the wildcards agree."""
1733             if self.ends_triple_dot != other_path.ends_triple_dot:
1734                  die("Both paths must end with ... if either does;\n" +
1735                      "paths: %s %s" % (self.path, other_path.path))
1736
1737         def match_wildcards(self, test_path):
1738             """See if this test_path matches us, and fill in the value
1739                of the wildcards if so.  Returns a tuple of
1740                (True|False, wildcards[]).  For now, only the ... at end
1741                is supported, so at most one wildcard."""
1742             if self.ends_triple_dot:
1743                 dotless = self.path[:-3]
1744                 if test_path.startswith(dotless):
1745                     wildcard = test_path[len(dotless):]
1746                     return (True, [ wildcard ])
1747             else:
1748                 if test_path == self.path:
1749                     return (True, [])
1750             return (False, [])
1751
1752         def match(self, test_path):
1753             """Just return if it matches; don't bother with the wildcards."""
1754             b, _ = self.match_wildcards(test_path)
1755             return b
1756
1757         def fill_in_wildcards(self, wildcards):
1758             """Return the relative path, with the wildcards filled in
1759                if there are any."""
1760             if self.ends_triple_dot:
1761                 return self.path[:-3] + wildcards[0]
1762             else:
1763                 return self.path
1764
1765     class Mapping(object):
1766         def __init__(self, depot_side, client_side, overlay, exclude):
1767             # depot_side is without the trailing /... if it had one
1768             self.depot_side = View.Path(depot_side, is_depot=True)
1769             self.client_side = View.Path(client_side, is_depot=False)
1770             self.overlay = overlay  # started with "+"
1771             self.exclude = exclude  # started with "-"
1772             assert not (self.overlay and self.exclude)
1773             self.depot_side.ensure_compatible(self.client_side)
1774
1775         def __str__(self):
1776             c = " "
1777             if self.overlay:
1778                 c = "+"
1779             if self.exclude:
1780                 c = "-"
1781             return "View.Mapping: %s%s -> %s" % \
1782                    (c, self.depot_side.path, self.client_side.path)
1783
1784         def map_depot_to_client(self, depot_path):
1785             """Calculate the client path if using this mapping on the
1786                given depot path; does not consider the effect of other
1787                mappings in a view.  Even excluded mappings are returned."""
1788             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1789             if not matches:
1790                 return ""
1791             client_path = self.client_side.fill_in_wildcards(wildcards)
1792             return client_path
1793
1794     #
1795     # View methods
1796     #
1797     def __init__(self):
1798         self.mappings = []
1799
1800     def append(self, view_line):
1801         """Parse a view line, splitting it into depot and client
1802            sides.  Append to self.mappings, preserving order."""
1803
1804         # Split the view line into exactly two words.  P4 enforces
1805         # structure on these lines that simplifies this quite a bit.
1806         #
1807         # Either or both words may be double-quoted.
1808         # Single quotes do not matter.
1809         # Double-quote marks cannot occur inside the words.
1810         # A + or - prefix is also inside the quotes.
1811         # There are no quotes unless they contain a space.
1812         # The line is already white-space stripped.
1813         # The two words are separated by a single space.
1814         #
1815         if view_line[0] == '"':
1816             # First word is double quoted.  Find its end.
1817             close_quote_index = view_line.find('"', 1)
1818             if close_quote_index <= 0:
1819                 die("No first-word closing quote found: %s" % view_line)
1820             depot_side = view_line[1:close_quote_index]
1821             # skip closing quote and space
1822             rhs_index = close_quote_index + 1 + 1
1823         else:
1824             space_index = view_line.find(" ")
1825             if space_index <= 0:
1826                 die("No word-splitting space found: %s" % view_line)
1827             depot_side = view_line[0:space_index]
1828             rhs_index = space_index + 1
1829
1830         if view_line[rhs_index] == '"':
1831             # Second word is double quoted.  Make sure there is a
1832             # double quote at the end too.
1833             if not view_line.endswith('"'):
1834                 die("View line with rhs quote should end with one: %s" %
1835                     view_line)
1836             # skip the quotes
1837             client_side = view_line[rhs_index+1:-1]
1838         else:
1839             client_side = view_line[rhs_index:]
1840
1841         # prefix + means overlay on previous mapping
1842         overlay = False
1843         if depot_side.startswith("+"):
1844             overlay = True
1845             depot_side = depot_side[1:]
1846
1847         # prefix - means exclude this path
1848         exclude = False
1849         if depot_side.startswith("-"):
1850             exclude = True
1851             depot_side = depot_side[1:]
1852
1853         m = View.Mapping(depot_side, client_side, overlay, exclude)
1854         self.mappings.append(m)
1855
1856     def map_in_client(self, depot_path):
1857         """Return the relative location in the client where this
1858            depot file should live.  Returns "" if the file should
1859            not be mapped in the client."""
1860
1861         paths_filled = []
1862         client_path = ""
1863
1864         # look at later entries first
1865         for m in self.mappings[::-1]:
1866
1867             # see where will this path end up in the client
1868             p = m.map_depot_to_client(depot_path)
1869
1870             if p == "":
1871                 # Depot path does not belong in client.  Must remember
1872                 # this, as previous items should not cause files to
1873                 # exist in this path either.  Remember that the list is
1874                 # being walked from the end, which has higher precedence.
1875                 # Overlap mappings do not exclude previous mappings.
1876                 if not m.overlay:
1877                     paths_filled.append(m.client_side)
1878
1879             else:
1880                 # This mapping matched; no need to search any further.
1881                 # But, the mapping could be rejected if the client path
1882                 # has already been claimed by an earlier mapping (i.e.
1883                 # one later in the list, which we are walking backwards).
1884                 already_mapped_in_client = False
1885                 for f in paths_filled:
1886                     # this is View.Path.match
1887                     if f.match(p):
1888                         already_mapped_in_client = True
1889                         break
1890                 if not already_mapped_in_client:
1891                     # Include this file, unless it is from a line that
1892                     # explicitly said to exclude it.
1893                     if not m.exclude:
1894                         client_path = p
1895
1896                 # a match, even if rejected, always stops the search
1897                 break
1898
1899         return client_path
1900
1901 class P4Sync(Command, P4UserMap):
1902     delete_actions = ( "delete", "move/delete", "purge" )
1903
1904     def __init__(self):
1905         Command.__init__(self)
1906         P4UserMap.__init__(self)
1907         self.options = [
1908                 optparse.make_option("--branch", dest="branch"),
1909                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1910                 optparse.make_option("--changesfile", dest="changesFile"),
1911                 optparse.make_option("--silent", dest="silent", action="store_true"),
1912                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1913                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1914                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1915                                      help="Import into refs/heads/ , not refs/remotes"),
1916                 optparse.make_option("--max-changes", dest="maxChanges"),
1917                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1918                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1919                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1920                                      help="Only sync files that are included in the Perforce Client Spec")
1921         ]
1922         self.description = """Imports from Perforce into a git repository.\n
1923     example:
1924     //depot/my/project/ -- to import the current head
1925     //depot/my/project/@all -- to import everything
1926     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1927
1928     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1929
1930         self.usage += " //depot/path[@revRange]"
1931         self.silent = False
1932         self.createdBranches = set()
1933         self.committedChanges = set()
1934         self.branch = ""
1935         self.detectBranches = False
1936         self.detectLabels = False
1937         self.importLabels = False
1938         self.changesFile = ""
1939         self.syncWithOrigin = True
1940         self.importIntoRemotes = True
1941         self.maxChanges = ""
1942         self.isWindows = (platform.system() == "Windows")
1943         self.keepRepoPath = False
1944         self.depotPaths = None
1945         self.p4BranchesInGit = []
1946         self.cloneExclude = []
1947         self.useClientSpec = False
1948         self.useClientSpec_from_options = False
1949         self.clientSpecDirs = None
1950         self.tempBranches = []
1951         self.tempBranchLocation = "git-p4-tmp"
1952
1953         if gitConfig("git-p4.syncFromOrigin") == "false":
1954             self.syncWithOrigin = False
1955
1956     # Force a checkpoint in fast-import and wait for it to finish
1957     def checkpoint(self):
1958         self.gitStream.write("checkpoint\n\n")
1959         self.gitStream.write("progress checkpoint\n\n")
1960         out = self.gitOutput.readline()
1961         if self.verbose:
1962             print "checkpoint finished: " + out
1963
1964     def extractFilesFromCommit(self, commit):
1965         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1966                              for path in self.cloneExclude]
1967         files = []
1968         fnum = 0
1969         while commit.has_key("depotFile%s" % fnum):
1970             path =  commit["depotFile%s" % fnum]
1971
1972             if [p for p in self.cloneExclude
1973                 if p4PathStartsWith(path, p)]:
1974                 found = False
1975             else:
1976                 found = [p for p in self.depotPaths
1977                          if p4PathStartsWith(path, p)]
1978             if not found:
1979                 fnum = fnum + 1
1980                 continue
1981
1982             file = {}
1983             file["path"] = path
1984             file["rev"] = commit["rev%s" % fnum]
1985             file["action"] = commit["action%s" % fnum]
1986             file["type"] = commit["type%s" % fnum]
1987             files.append(file)
1988             fnum = fnum + 1
1989         return files
1990
1991     def stripRepoPath(self, path, prefixes):
1992         """When streaming files, this is called to map a p4 depot path
1993            to where it should go in git.  The prefixes are either
1994            self.depotPaths, or self.branchPrefixes in the case of
1995            branch detection."""
1996
1997         if self.useClientSpec:
1998             # branch detection moves files up a level (the branch name)
1999             # from what client spec interpretation gives
2000             path = self.clientSpecDirs.map_in_client(path)
2001             if self.detectBranches:
2002                 for b in self.knownBranches:
2003                     if path.startswith(b + "/"):
2004                         path = path[len(b)+1:]
2005
2006         elif self.keepRepoPath:
2007             # Preserve everything in relative path name except leading
2008             # //depot/; just look at first prefix as they all should
2009             # be in the same depot.
2010             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2011             if p4PathStartsWith(path, depot):
2012                 path = path[len(depot):]
2013
2014         else:
2015             for p in prefixes:
2016                 if p4PathStartsWith(path, p):
2017                     path = path[len(p):]
2018                     break
2019
2020         path = wildcard_decode(path)
2021         return path
2022
2023     def splitFilesIntoBranches(self, commit):
2024         """Look at each depotFile in the commit to figure out to what
2025            branch it belongs."""
2026
2027         branches = {}
2028         fnum = 0
2029         while commit.has_key("depotFile%s" % fnum):
2030             path =  commit["depotFile%s" % fnum]
2031             found = [p for p in self.depotPaths
2032                      if p4PathStartsWith(path, p)]
2033             if not found:
2034                 fnum = fnum + 1
2035                 continue
2036
2037             file = {}
2038             file["path"] = path
2039             file["rev"] = commit["rev%s" % fnum]
2040             file["action"] = commit["action%s" % fnum]
2041             file["type"] = commit["type%s" % fnum]
2042             fnum = fnum + 1
2043
2044             # start with the full relative path where this file would
2045             # go in a p4 client
2046             if self.useClientSpec:
2047                 relPath = self.clientSpecDirs.map_in_client(path)
2048             else:
2049                 relPath = self.stripRepoPath(path, self.depotPaths)
2050
2051             for branch in self.knownBranches.keys():
2052                 # add a trailing slash so that a commit into qt/4.2foo
2053                 # doesn't end up in qt/4.2, e.g.
2054                 if relPath.startswith(branch + "/"):
2055                     if branch not in branches:
2056                         branches[branch] = []
2057                     branches[branch].append(file)
2058                     break
2059
2060         return branches
2061
2062     # output one file from the P4 stream
2063     # - helper for streamP4Files
2064
2065     def streamOneP4File(self, file, contents):
2066         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2067         if verbose:
2068             sys.stderr.write("%s\n" % relPath)
2069
2070         (type_base, type_mods) = split_p4_type(file["type"])
2071
2072         git_mode = "100644"
2073         if "x" in type_mods:
2074             git_mode = "100755"
2075         if type_base == "symlink":
2076             git_mode = "120000"
2077             # p4 print on a symlink contains "target\n"; remove the newline
2078             data = ''.join(contents)
2079             contents = [data[:-1]]
2080
2081         if type_base == "utf16":
2082             # p4 delivers different text in the python output to -G
2083             # than it does when using "print -o", or normal p4 client
2084             # operations.  utf16 is converted to ascii or utf8, perhaps.
2085             # But ascii text saved as -t utf16 is completely mangled.
2086             # Invoke print -o to get the real contents.
2087             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2088             contents = [ text ]
2089
2090         if type_base == "apple":
2091             # Apple filetype files will be streamed as a concatenation of
2092             # its appledouble header and the contents.  This is useless
2093             # on both macs and non-macs.  If using "print -q -o xx", it
2094             # will create "xx" with the data, and "%xx" with the header.
2095             # This is also not very useful.
2096             #
2097             # Ideally, someday, this script can learn how to generate
2098             # appledouble files directly and import those to git, but
2099             # non-mac machines can never find a use for apple filetype.
2100             print "\nIgnoring apple filetype file %s" % file['depotFile']
2101             return
2102
2103         # Perhaps windows wants unicode, utf16 newlines translated too;
2104         # but this is not doing it.
2105         if self.isWindows and type_base == "text":
2106             mangled = []
2107             for data in contents:
2108                 data = data.replace("\r\n", "\n")
2109                 mangled.append(data)
2110             contents = mangled
2111
2112         # Note that we do not try to de-mangle keywords on utf16 files,
2113         # even though in theory somebody may want that.
2114         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2115         if pattern:
2116             regexp = re.compile(pattern, re.VERBOSE)
2117             text = ''.join(contents)
2118             text = regexp.sub(r'$\1$', text)
2119             contents = [ text ]
2120
2121         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2122
2123         # total length...
2124         length = 0
2125         for d in contents:
2126             length = length + len(d)
2127
2128         self.gitStream.write("data %d\n" % length)
2129         for d in contents:
2130             self.gitStream.write(d)
2131         self.gitStream.write("\n")
2132
2133     def streamOneP4Deletion(self, file):
2134         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2135         if verbose:
2136             sys.stderr.write("delete %s\n" % relPath)
2137         self.gitStream.write("D %s\n" % relPath)
2138
2139     # handle another chunk of streaming data
2140     def streamP4FilesCb(self, marshalled):
2141
2142         # catch p4 errors and complain
2143         err = None
2144         if "code" in marshalled:
2145             if marshalled["code"] == "error":
2146                 if "data" in marshalled:
2147                     err = marshalled["data"].rstrip()
2148         if err:
2149             f = None
2150             if self.stream_have_file_info:
2151                 if "depotFile" in self.stream_file:
2152                     f = self.stream_file["depotFile"]
2153             # force a failure in fast-import, else an empty
2154             # commit will be made
2155             self.gitStream.write("\n")
2156             self.gitStream.write("die-now\n")
2157             self.gitStream.close()
2158             # ignore errors, but make sure it exits first
2159             self.importProcess.wait()
2160             if f:
2161                 die("Error from p4 print for %s: %s" % (f, err))
2162             else:
2163                 die("Error from p4 print: %s" % err)
2164
2165         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2166             # start of a new file - output the old one first
2167             self.streamOneP4File(self.stream_file, self.stream_contents)
2168             self.stream_file = {}
2169             self.stream_contents = []
2170             self.stream_have_file_info = False
2171
2172         # pick up the new file information... for the
2173         # 'data' field we need to append to our array
2174         for k in marshalled.keys():
2175             if k == 'data':
2176                 self.stream_contents.append(marshalled['data'])
2177             else:
2178                 self.stream_file[k] = marshalled[k]
2179
2180         self.stream_have_file_info = True
2181
2182     # Stream directly from "p4 files" into "git fast-import"
2183     def streamP4Files(self, files):
2184         filesForCommit = []
2185         filesToRead = []
2186         filesToDelete = []
2187
2188         for f in files:
2189             # if using a client spec, only add the files that have
2190             # a path in the client
2191             if self.clientSpecDirs:
2192                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2193                     continue
2194
2195             filesForCommit.append(f)
2196             if f['action'] in self.delete_actions:
2197                 filesToDelete.append(f)
2198             else:
2199                 filesToRead.append(f)
2200
2201         # deleted files...
2202         for f in filesToDelete:
2203             self.streamOneP4Deletion(f)
2204
2205         if len(filesToRead) > 0:
2206             self.stream_file = {}
2207             self.stream_contents = []
2208             self.stream_have_file_info = False
2209
2210             # curry self argument
2211             def streamP4FilesCbSelf(entry):
2212                 self.streamP4FilesCb(entry)
2213
2214             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2215
2216             p4CmdList(["-x", "-", "print"],
2217                       stdin=fileArgs,
2218                       cb=streamP4FilesCbSelf)
2219
2220             # do the last chunk
2221             if self.stream_file.has_key('depotFile'):
2222                 self.streamOneP4File(self.stream_file, self.stream_contents)
2223
2224     def make_email(self, userid):
2225         if userid in self.users:
2226             return self.users[userid]
2227         else:
2228             return "%s <a@b>" % userid
2229
2230     # Stream a p4 tag
2231     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2232         if verbose:
2233             print "writing tag %s for commit %s" % (labelName, commit)
2234         gitStream.write("tag %s\n" % labelName)
2235         gitStream.write("from %s\n" % commit)
2236
2237         if labelDetails.has_key('Owner'):
2238             owner = labelDetails["Owner"]
2239         else:
2240             owner = None
2241
2242         # Try to use the owner of the p4 label, or failing that,
2243         # the current p4 user id.
2244         if owner:
2245             email = self.make_email(owner)
2246         else:
2247             email = self.make_email(self.p4UserId())
2248         tagger = "%s %s %s" % (email, epoch, self.tz)
2249
2250         gitStream.write("tagger %s\n" % tagger)
2251
2252         print "labelDetails=",labelDetails
2253         if labelDetails.has_key('Description'):
2254             description = labelDetails['Description']
2255         else:
2256             description = 'Label from git p4'
2257
2258         gitStream.write("data %d\n" % len(description))
2259         gitStream.write(description)
2260         gitStream.write("\n")
2261
2262     def commit(self, details, files, branch, parent = ""):
2263         epoch = details["time"]
2264         author = details["user"]
2265
2266         if self.verbose:
2267             print "commit into %s" % branch
2268
2269         # start with reading files; if that fails, we should not
2270         # create a commit.
2271         new_files = []
2272         for f in files:
2273             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2274                 new_files.append (f)
2275             else:
2276                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2277
2278         self.gitStream.write("commit %s\n" % branch)
2279 #        gitStream.write("mark :%s\n" % details["change"])
2280         self.committedChanges.add(int(details["change"]))
2281         committer = ""
2282         if author not in self.users:
2283             self.getUserMapFromPerforceServer()
2284         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2285
2286         self.gitStream.write("committer %s\n" % committer)
2287
2288         self.gitStream.write("data <<EOT\n")
2289         self.gitStream.write(details["desc"])
2290         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2291                              (','.join(self.branchPrefixes), details["change"]))
2292         if len(details['options']) > 0:
2293             self.gitStream.write(": options = %s" % details['options'])
2294         self.gitStream.write("]\nEOT\n\n")
2295
2296         if len(parent) > 0:
2297             if self.verbose:
2298                 print "parent %s" % parent
2299             self.gitStream.write("from %s\n" % parent)
2300
2301         self.streamP4Files(new_files)
2302         self.gitStream.write("\n")
2303
2304         change = int(details["change"])
2305
2306         if self.labels.has_key(change):
2307             label = self.labels[change]
2308             labelDetails = label[0]
2309             labelRevisions = label[1]
2310             if self.verbose:
2311                 print "Change %s is labelled %s" % (change, labelDetails)
2312
2313             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2314                                                 for p in self.branchPrefixes])
2315
2316             if len(files) == len(labelRevisions):
2317
2318                 cleanedFiles = {}
2319                 for info in files:
2320                     if info["action"] in self.delete_actions:
2321                         continue
2322                     cleanedFiles[info["depotFile"]] = info["rev"]
2323
2324                 if cleanedFiles == labelRevisions:
2325                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2326
2327                 else:
2328                     if not self.silent:
2329                         print ("Tag %s does not match with change %s: files do not match."
2330                                % (labelDetails["label"], change))
2331
2332             else:
2333                 if not self.silent:
2334                     print ("Tag %s does not match with change %s: file count is different."
2335                            % (labelDetails["label"], change))
2336
2337     # Build a dictionary of changelists and labels, for "detect-labels" option.
2338     def getLabels(self):
2339         self.labels = {}
2340
2341         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2342         if len(l) > 0 and not self.silent:
2343             print "Finding files belonging to labels in %s" % `self.depotPaths`
2344
2345         for output in l:
2346             label = output["label"]
2347             revisions = {}
2348             newestChange = 0
2349             if self.verbose:
2350                 print "Querying files for label %s" % label
2351             for file in p4CmdList(["files"] +
2352                                       ["%s...@%s" % (p, label)
2353                                           for p in self.depotPaths]):
2354                 revisions[file["depotFile"]] = file["rev"]
2355                 change = int(file["change"])
2356                 if change > newestChange:
2357                     newestChange = change
2358
2359             self.labels[newestChange] = [output, revisions]
2360
2361         if self.verbose:
2362             print "Label changes: %s" % self.labels.keys()
2363
2364     # Import p4 labels as git tags. A direct mapping does not
2365     # exist, so assume that if all the files are at the same revision
2366     # then we can use that, or it's something more complicated we should
2367     # just ignore.
2368     def importP4Labels(self, stream, p4Labels):
2369         if verbose:
2370             print "import p4 labels: " + ' '.join(p4Labels)
2371
2372         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2373         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2374         if len(validLabelRegexp) == 0:
2375             validLabelRegexp = defaultLabelRegexp
2376         m = re.compile(validLabelRegexp)
2377
2378         for name in p4Labels:
2379             commitFound = False
2380
2381             if not m.match(name):
2382                 if verbose:
2383                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2384                 continue
2385
2386             if name in ignoredP4Labels:
2387                 continue
2388
2389             labelDetails = p4CmdList(['label', "-o", name])[0]
2390
2391             # get the most recent changelist for each file in this label
2392             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2393                                 for p in self.depotPaths])
2394
2395             if change.has_key('change'):
2396                 # find the corresponding git commit; take the oldest commit
2397                 changelist = int(change['change'])
2398                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2399                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2400                 if len(gitCommit) == 0:
2401                     print "could not find git commit for changelist %d" % changelist
2402                 else:
2403                     gitCommit = gitCommit.strip()
2404                     commitFound = True
2405                     # Convert from p4 time format
2406                     try:
2407                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2408                     except ValueError:
2409                         print "Could not convert label time %s" % labelDetails['Update']
2410                         tmwhen = 1
2411
2412                     when = int(time.mktime(tmwhen))
2413                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2414                     if verbose:
2415                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2416             else:
2417                 if verbose:
2418                     print "Label %s has no changelists - possibly deleted?" % name
2419
2420             if not commitFound:
2421                 # We can't import this label; don't try again as it will get very
2422                 # expensive repeatedly fetching all the files for labels that will
2423                 # never be imported. If the label is moved in the future, the
2424                 # ignore will need to be removed manually.
2425                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2426
2427     def guessProjectName(self):
2428         for p in self.depotPaths:
2429             if p.endswith("/"):
2430                 p = p[:-1]
2431             p = p[p.strip().rfind("/") + 1:]
2432             if not p.endswith("/"):
2433                p += "/"
2434             return p
2435
2436     def getBranchMapping(self):
2437         lostAndFoundBranches = set()
2438
2439         user = gitConfig("git-p4.branchUser")
2440         if len(user) > 0:
2441             command = "branches -u %s" % user
2442         else:
2443             command = "branches"
2444
2445         for info in p4CmdList(command):
2446             details = p4Cmd(["branch", "-o", info["branch"]])
2447             viewIdx = 0
2448             while details.has_key("View%s" % viewIdx):
2449                 paths = details["View%s" % viewIdx].split(" ")
2450                 viewIdx = viewIdx + 1
2451                 # require standard //depot/foo/... //depot/bar/... mapping
2452                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2453                     continue
2454                 source = paths[0]
2455                 destination = paths[1]
2456                 ## HACK
2457                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2458                     source = source[len(self.depotPaths[0]):-4]
2459                     destination = destination[len(self.depotPaths[0]):-4]
2460
2461                     if destination in self.knownBranches:
2462                         if not self.silent:
2463                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2464                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2465                         continue
2466
2467                     self.knownBranches[destination] = source
2468
2469                     lostAndFoundBranches.discard(destination)
2470
2471                     if source not in self.knownBranches:
2472                         lostAndFoundBranches.add(source)
2473
2474         # Perforce does not strictly require branches to be defined, so we also
2475         # check git config for a branch list.
2476         #
2477         # Example of branch definition in git config file:
2478         # [git-p4]
2479         #   branchList=main:branchA
2480         #   branchList=main:branchB
2481         #   branchList=branchA:branchC
2482         configBranches = gitConfigList("git-p4.branchList")
2483         for branch in configBranches:
2484             if branch:
2485                 (source, destination) = branch.split(":")
2486                 self.knownBranches[destination] = source
2487
2488                 lostAndFoundBranches.discard(destination)
2489
2490                 if source not in self.knownBranches:
2491                     lostAndFoundBranches.add(source)
2492
2493
2494         for branch in lostAndFoundBranches:
2495             self.knownBranches[branch] = branch
2496
2497     def getBranchMappingFromGitBranches(self):
2498         branches = p4BranchesInGit(self.importIntoRemotes)
2499         for branch in branches.keys():
2500             if branch == "master":
2501                 branch = "main"
2502             else:
2503                 branch = branch[len(self.projectName):]
2504             self.knownBranches[branch] = branch
2505
2506     def listExistingP4GitBranches(self):
2507         # branches holds mapping from name to commit
2508         branches = p4BranchesInGit(self.importIntoRemotes)
2509         self.p4BranchesInGit = branches.keys()
2510         for branch in branches.keys():
2511             self.initialParents[self.refPrefix + branch] = branches[branch]
2512
2513     def updateOptionDict(self, d):
2514         option_keys = {}
2515         if self.keepRepoPath:
2516             option_keys['keepRepoPath'] = 1
2517
2518         d["options"] = ' '.join(sorted(option_keys.keys()))
2519
2520     def readOptions(self, d):
2521         self.keepRepoPath = (d.has_key('options')
2522                              and ('keepRepoPath' in d['options']))
2523
2524     def gitRefForBranch(self, branch):
2525         if branch == "main":
2526             return self.refPrefix + "master"
2527
2528         if len(branch) <= 0:
2529             return branch
2530
2531         return self.refPrefix + self.projectName + branch
2532
2533     def gitCommitByP4Change(self, ref, change):
2534         if self.verbose:
2535             print "looking in ref " + ref + " for change %s using bisect..." % change
2536
2537         earliestCommit = ""
2538         latestCommit = parseRevision(ref)
2539
2540         while True:
2541             if self.verbose:
2542                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2543             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2544             if len(next) == 0:
2545                 if self.verbose:
2546                     print "argh"
2547                 return ""
2548             log = extractLogMessageFromGitCommit(next)
2549             settings = extractSettingsGitLog(log)
2550             currentChange = int(settings['change'])
2551             if self.verbose:
2552                 print "current change %s" % currentChange
2553
2554             if currentChange == change:
2555                 if self.verbose:
2556                     print "found %s" % next
2557                 return next
2558
2559             if currentChange < change:
2560                 earliestCommit = "^%s" % next
2561             else:
2562                 latestCommit = "%s" % next
2563
2564         return ""
2565
2566     def importNewBranch(self, branch, maxChange):
2567         # make fast-import flush all changes to disk and update the refs using the checkpoint
2568         # command so that we can try to find the branch parent in the git history
2569         self.gitStream.write("checkpoint\n\n");
2570         self.gitStream.flush();
2571         branchPrefix = self.depotPaths[0] + branch + "/"
2572         range = "@1,%s" % maxChange
2573         #print "prefix" + branchPrefix
2574         changes = p4ChangesForPaths([branchPrefix], range)
2575         if len(changes) <= 0:
2576             return False
2577         firstChange = changes[0]
2578         #print "first change in branch: %s" % firstChange
2579         sourceBranch = self.knownBranches[branch]
2580         sourceDepotPath = self.depotPaths[0] + sourceBranch
2581         sourceRef = self.gitRefForBranch(sourceBranch)
2582         #print "source " + sourceBranch
2583
2584         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2585         #print "branch parent: %s" % branchParentChange
2586         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2587         if len(gitParent) > 0:
2588             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2589             #print "parent git commit: %s" % gitParent
2590
2591         self.importChanges(changes)
2592         return True
2593
2594     def searchParent(self, parent, branch, target):
2595         parentFound = False
2596         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2597             blob = blob.strip()
2598             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2599                 parentFound = True
2600                 if self.verbose:
2601                     print "Found parent of %s in commit %s" % (branch, blob)
2602                 break
2603         if parentFound:
2604             return blob
2605         else:
2606             return None
2607
2608     def importChanges(self, changes):
2609         cnt = 1
2610         for change in changes:
2611             description = p4_describe(change)
2612             self.updateOptionDict(description)
2613
2614             if not self.silent:
2615                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2616                 sys.stdout.flush()
2617             cnt = cnt + 1
2618
2619             try:
2620                 if self.detectBranches:
2621                     branches = self.splitFilesIntoBranches(description)
2622                     for branch in branches.keys():
2623                         ## HACK  --hwn
2624                         branchPrefix = self.depotPaths[0] + branch + "/"
2625                         self.branchPrefixes = [ branchPrefix ]
2626
2627                         parent = ""
2628
2629                         filesForCommit = branches[branch]
2630
2631                         if self.verbose:
2632                             print "branch is %s" % branch
2633
2634                         self.updatedBranches.add(branch)
2635
2636                         if branch not in self.createdBranches:
2637                             self.createdBranches.add(branch)
2638                             parent = self.knownBranches[branch]
2639                             if parent == branch:
2640                                 parent = ""
2641                             else:
2642                                 fullBranch = self.projectName + branch
2643                                 if fullBranch not in self.p4BranchesInGit:
2644                                     if not self.silent:
2645                                         print("\n    Importing new branch %s" % fullBranch);
2646                                     if self.importNewBranch(branch, change - 1):
2647                                         parent = ""
2648                                         self.p4BranchesInGit.append(fullBranch)
2649                                     if not self.silent:
2650                                         print("\n    Resuming with change %s" % change);
2651
2652                                 if self.verbose:
2653                                     print "parent determined through known branches: %s" % parent
2654
2655                         branch = self.gitRefForBranch(branch)
2656                         parent = self.gitRefForBranch(parent)
2657
2658                         if self.verbose:
2659                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2660
2661                         if len(parent) == 0 and branch in self.initialParents:
2662                             parent = self.initialParents[branch]
2663                             del self.initialParents[branch]
2664
2665                         blob = None
2666                         if len(parent) > 0:
2667                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2668                             if self.verbose:
2669                                 print "Creating temporary branch: " + tempBranch
2670                             self.commit(description, filesForCommit, tempBranch)
2671                             self.tempBranches.append(tempBranch)
2672                             self.checkpoint()
2673                             blob = self.searchParent(parent, branch, tempBranch)
2674                         if blob:
2675                             self.commit(description, filesForCommit, branch, blob)
2676                         else:
2677                             if self.verbose:
2678                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2679                             self.commit(description, filesForCommit, branch, parent)
2680                 else:
2681                     files = self.extractFilesFromCommit(description)
2682                     self.commit(description, files, self.branch,
2683                                 self.initialParent)
2684                     self.initialParent = ""
2685             except IOError:
2686                 print self.gitError.read()
2687                 sys.exit(1)
2688
2689     def importHeadRevision(self, revision):
2690         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2691
2692         details = {}
2693         details["user"] = "git perforce import user"
2694         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2695                            % (' '.join(self.depotPaths), revision))
2696         details["change"] = revision
2697         newestRevision = 0
2698
2699         fileCnt = 0
2700         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2701
2702         for info in p4CmdList(["files"] + fileArgs):
2703
2704             if 'code' in info and info['code'] == 'error':
2705                 sys.stderr.write("p4 returned an error: %s\n"
2706                                  % info['data'])
2707                 if info['data'].find("must refer to client") >= 0:
2708                     sys.stderr.write("This particular p4 error is misleading.\n")
2709                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2710                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2711                 sys.exit(1)
2712             if 'p4ExitCode' in info:
2713                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2714                 sys.exit(1)
2715
2716
2717             change = int(info["change"])
2718             if change > newestRevision:
2719                 newestRevision = change
2720
2721             if info["action"] in self.delete_actions:
2722                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2723                 #fileCnt = fileCnt + 1
2724                 continue
2725
2726             for prop in ["depotFile", "rev", "action", "type" ]:
2727                 details["%s%s" % (prop, fileCnt)] = info[prop]
2728
2729             fileCnt = fileCnt + 1
2730
2731         details["change"] = newestRevision
2732
2733         # Use time from top-most change so that all git p4 clones of
2734         # the same p4 repo have the same commit SHA1s.
2735         res = p4_describe(newestRevision)
2736         details["time"] = res["time"]
2737
2738         self.updateOptionDict(details)
2739         try:
2740             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2741         except IOError:
2742             print "IO error with git fast-import. Is your git version recent enough?"
2743             print self.gitError.read()
2744
2745
2746     def run(self, args):
2747         self.depotPaths = []
2748         self.changeRange = ""
2749         self.initialParent = ""
2750         self.previousDepotPaths = []
2751
2752         # map from branch depot path to parent branch
2753         self.knownBranches = {}
2754         self.initialParents = {}
2755         self.hasOrigin = originP4BranchesExist()
2756         if not self.syncWithOrigin:
2757             self.hasOrigin = False
2758
2759         if self.importIntoRemotes:
2760             self.refPrefix = "refs/remotes/p4/"
2761         else:
2762             self.refPrefix = "refs/heads/p4/"
2763
2764         if self.syncWithOrigin and self.hasOrigin:
2765             if not self.silent:
2766                 print "Syncing with origin first by calling git fetch origin"
2767             system("git fetch origin")
2768
2769         if len(self.branch) == 0:
2770             self.branch = self.refPrefix + "master"
2771             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2772                 system("git update-ref %s refs/heads/p4" % self.branch)
2773                 system("git branch -D p4");
2774             # create it /after/ importing, when master exists
2775             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2776                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2777
2778         # accept either the command-line option, or the configuration variable
2779         if self.useClientSpec:
2780             # will use this after clone to set the variable
2781             self.useClientSpec_from_options = True
2782         else:
2783             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2784                 self.useClientSpec = True
2785         if self.useClientSpec:
2786             self.clientSpecDirs = getClientSpec()
2787
2788         # TODO: should always look at previous commits,
2789         # merge with previous imports, if possible.
2790         if args == []:
2791             if self.hasOrigin:
2792                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2793             self.listExistingP4GitBranches()
2794
2795             if len(self.p4BranchesInGit) > 1:
2796                 if not self.silent:
2797                     print "Importing from/into multiple branches"
2798                 self.detectBranches = True
2799
2800             if self.verbose:
2801                 print "branches: %s" % self.p4BranchesInGit
2802
2803             p4Change = 0
2804             for branch in self.p4BranchesInGit:
2805                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2806
2807                 settings = extractSettingsGitLog(logMsg)
2808
2809                 self.readOptions(settings)
2810                 if (settings.has_key('depot-paths')
2811                     and settings.has_key ('change')):
2812                     change = int(settings['change']) + 1
2813                     p4Change = max(p4Change, change)
2814
2815                     depotPaths = sorted(settings['depot-paths'])
2816                     if self.previousDepotPaths == []:
2817                         self.previousDepotPaths = depotPaths
2818                     else:
2819                         paths = []
2820                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2821                             prev_list = prev.split("/")
2822                             cur_list = cur.split("/")
2823                             for i in range(0, min(len(cur_list), len(prev_list))):
2824                                 if cur_list[i] <> prev_list[i]:
2825                                     i = i - 1
2826                                     break
2827
2828                             paths.append ("/".join(cur_list[:i + 1]))
2829
2830                         self.previousDepotPaths = paths
2831
2832             if p4Change > 0:
2833                 self.depotPaths = sorted(self.previousDepotPaths)
2834                 self.changeRange = "@%s,#head" % p4Change
2835                 if not self.detectBranches:
2836                     self.initialParent = parseRevision(self.branch)
2837                 if not self.silent and not self.detectBranches:
2838                     print "Performing incremental import into %s git branch" % self.branch
2839
2840         if not self.branch.startswith("refs/"):
2841             self.branch = "refs/heads/" + self.branch
2842
2843         if len(args) == 0 and self.depotPaths:
2844             if not self.silent:
2845                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2846         else:
2847             if self.depotPaths and self.depotPaths != args:
2848                 print ("previous import used depot path %s and now %s was specified. "
2849                        "This doesn't work!" % (' '.join (self.depotPaths),
2850                                                ' '.join (args)))
2851                 sys.exit(1)
2852
2853             self.depotPaths = sorted(args)
2854
2855         revision = ""
2856         self.users = {}
2857
2858         # Make sure no revision specifiers are used when --changesfile
2859         # is specified.
2860         bad_changesfile = False
2861         if len(self.changesFile) > 0:
2862             for p in self.depotPaths:
2863                 if p.find("@") >= 0 or p.find("#") >= 0:
2864                     bad_changesfile = True
2865                     break
2866         if bad_changesfile:
2867             die("Option --changesfile is incompatible with revision specifiers")
2868
2869         newPaths = []
2870         for p in self.depotPaths:
2871             if p.find("@") != -1:
2872                 atIdx = p.index("@")
2873                 self.changeRange = p[atIdx:]
2874                 if self.changeRange == "@all":
2875                     self.changeRange = ""
2876                 elif ',' not in self.changeRange:
2877                     revision = self.changeRange
2878                     self.changeRange = ""
2879                 p = p[:atIdx]
2880             elif p.find("#") != -1:
2881                 hashIdx = p.index("#")
2882                 revision = p[hashIdx:]
2883                 p = p[:hashIdx]
2884             elif self.previousDepotPaths == []:
2885                 # pay attention to changesfile, if given, else import
2886                 # the entire p4 tree at the head revision
2887                 if len(self.changesFile) == 0:
2888                     revision = "#head"
2889
2890             p = re.sub ("\.\.\.$", "", p)
2891             if not p.endswith("/"):
2892                 p += "/"
2893
2894             newPaths.append(p)
2895
2896         self.depotPaths = newPaths
2897
2898         # --detect-branches may change this for each branch
2899         self.branchPrefixes = self.depotPaths
2900
2901         self.loadUserMapFromCache()
2902         self.labels = {}
2903         if self.detectLabels:
2904             self.getLabels();
2905
2906         if self.detectBranches:
2907             ## FIXME - what's a P4 projectName ?
2908             self.projectName = self.guessProjectName()
2909
2910             if self.hasOrigin:
2911                 self.getBranchMappingFromGitBranches()
2912             else:
2913                 self.getBranchMapping()
2914             if self.verbose:
2915                 print "p4-git branches: %s" % self.p4BranchesInGit
2916                 print "initial parents: %s" % self.initialParents
2917             for b in self.p4BranchesInGit:
2918                 if b != "master":
2919
2920                     ## FIXME
2921                     b = b[len(self.projectName):]
2922                 self.createdBranches.add(b)
2923
2924         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2925
2926         self.importProcess = subprocess.Popen(["git", "fast-import"],
2927                                               stdin=subprocess.PIPE,
2928                                               stdout=subprocess.PIPE,
2929                                               stderr=subprocess.PIPE);
2930         self.gitOutput = self.importProcess.stdout
2931         self.gitStream = self.importProcess.stdin
2932         self.gitError = self.importProcess.stderr
2933
2934         if revision:
2935             self.importHeadRevision(revision)
2936         else:
2937             changes = []
2938
2939             if len(self.changesFile) > 0:
2940                 output = open(self.changesFile).readlines()
2941                 changeSet = set()
2942                 for line in output:
2943                     changeSet.add(int(line))
2944
2945                 for change in changeSet:
2946                     changes.append(change)
2947
2948                 changes.sort()
2949             else:
2950                 # catch "git p4 sync" with no new branches, in a repo that
2951                 # does not have any existing p4 branches
2952                 if len(args) == 0 and not self.p4BranchesInGit:
2953                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2954                 if self.verbose:
2955                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2956                                                               self.changeRange)
2957                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2958
2959                 if len(self.maxChanges) > 0:
2960                     changes = changes[:min(int(self.maxChanges), len(changes))]
2961
2962             if len(changes) == 0:
2963                 if not self.silent:
2964                     print "No changes to import!"
2965             else:
2966                 if not self.silent and not self.detectBranches:
2967                     print "Import destination: %s" % self.branch
2968
2969                 self.updatedBranches = set()
2970
2971                 self.importChanges(changes)
2972
2973                 if not self.silent:
2974                     print ""
2975                     if len(self.updatedBranches) > 0:
2976                         sys.stdout.write("Updated branches: ")
2977                         for b in self.updatedBranches:
2978                             sys.stdout.write("%s " % b)
2979                         sys.stdout.write("\n")
2980
2981         if gitConfig("git-p4.importLabels", "--bool") == "true":
2982             self.importLabels = True
2983
2984         if self.importLabels:
2985             p4Labels = getP4Labels(self.depotPaths)
2986             gitTags = getGitTags()
2987
2988             missingP4Labels = p4Labels - gitTags
2989             self.importP4Labels(self.gitStream, missingP4Labels)
2990
2991         self.gitStream.close()
2992         if self.importProcess.wait() != 0:
2993             die("fast-import failed: %s" % self.gitError.read())
2994         self.gitOutput.close()
2995         self.gitError.close()
2996
2997         # Cleanup temporary branches created during import
2998         if self.tempBranches != []:
2999             for branch in self.tempBranches:
3000                 read_pipe("git update-ref -d %s" % branch)
3001             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3002
3003         return True
3004
3005 class P4Rebase(Command):
3006     def __init__(self):
3007         Command.__init__(self)
3008         self.options = [
3009                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3010         ]
3011         self.importLabels = False
3012         self.description = ("Fetches the latest revision from perforce and "
3013                             + "rebases the current work (branch) against it")
3014
3015     def run(self, args):
3016         sync = P4Sync()
3017         sync.importLabels = self.importLabels
3018         sync.run([])
3019
3020         return self.rebase()
3021
3022     def rebase(self):
3023         if os.system("git update-index --refresh") != 0:
3024             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3025         if len(read_pipe("git diff-index HEAD --")) > 0:
3026             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3027
3028         [upstream, settings] = findUpstreamBranchPoint()
3029         if len(upstream) == 0:
3030             die("Cannot find upstream branchpoint for rebase")
3031
3032         # the branchpoint may be p4/foo~3, so strip off the parent
3033         upstream = re.sub("~[0-9]+$", "", upstream)
3034
3035         print "Rebasing the current branch onto %s" % upstream
3036         oldHead = read_pipe("git rev-parse HEAD").strip()
3037         system("git rebase %s" % upstream)
3038         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3039         return True
3040
3041 class P4Clone(P4Sync):
3042     def __init__(self):
3043         P4Sync.__init__(self)
3044         self.description = "Creates a new git repository and imports from Perforce into it"
3045         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3046         self.options += [
3047             optparse.make_option("--destination", dest="cloneDestination",
3048                                  action='store', default=None,
3049                                  help="where to leave result of the clone"),
3050             optparse.make_option("-/", dest="cloneExclude",
3051                                  action="append", type="string",
3052                                  help="exclude depot path"),
3053             optparse.make_option("--bare", dest="cloneBare",
3054                                  action="store_true", default=False),
3055         ]
3056         self.cloneDestination = None
3057         self.needsGit = False
3058         self.cloneBare = False
3059
3060     # This is required for the "append" cloneExclude action
3061     def ensure_value(self, attr, value):
3062         if not hasattr(self, attr) or getattr(self, attr) is None:
3063             setattr(self, attr, value)
3064         return getattr(self, attr)
3065
3066     def defaultDestination(self, args):
3067         ## TODO: use common prefix of args?
3068         depotPath = args[0]
3069         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3070         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3071         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3072         depotDir = re.sub(r"/$", "", depotDir)
3073         return os.path.split(depotDir)[1]
3074
3075     def run(self, args):
3076         if len(args) < 1:
3077             return False
3078
3079         if self.keepRepoPath and not self.cloneDestination:
3080             sys.stderr.write("Must specify destination for --keep-path\n")
3081             sys.exit(1)
3082
3083         depotPaths = args
3084
3085         if not self.cloneDestination and len(depotPaths) > 1:
3086             self.cloneDestination = depotPaths[-1]
3087             depotPaths = depotPaths[:-1]
3088
3089         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3090         for p in depotPaths:
3091             if not p.startswith("//"):
3092                 return False
3093
3094         if not self.cloneDestination:
3095             self.cloneDestination = self.defaultDestination(args)
3096
3097         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3098
3099         if not os.path.exists(self.cloneDestination):
3100             os.makedirs(self.cloneDestination)
3101         chdir(self.cloneDestination)
3102
3103         init_cmd = [ "git", "init" ]
3104         if self.cloneBare:
3105             init_cmd.append("--bare")
3106         subprocess.check_call(init_cmd)
3107
3108         if not P4Sync.run(self, depotPaths):
3109             return False
3110         if self.branch != "master":
3111             if self.importIntoRemotes:
3112                 masterbranch = "refs/remotes/p4/master"
3113             else:
3114                 masterbranch = "refs/heads/p4/master"
3115             if gitBranchExists(masterbranch):
3116                 system("git branch master %s" % masterbranch)
3117                 if not self.cloneBare:
3118                     system("git checkout -f")
3119             else:
3120                 print "Could not detect main branch. No checkout/master branch created."
3121
3122         # auto-set this variable if invoked with --use-client-spec
3123         if self.useClientSpec_from_options:
3124             system("git config --bool git-p4.useclientspec true")
3125
3126         return True
3127
3128 class P4Branches(Command):
3129     def __init__(self):
3130         Command.__init__(self)
3131         self.options = [ ]
3132         self.description = ("Shows the git branches that hold imports and their "
3133                             + "corresponding perforce depot paths")
3134         self.verbose = False
3135
3136     def run(self, args):
3137         if originP4BranchesExist():
3138             createOrUpdateBranchesFromOrigin()
3139
3140         cmdline = "git rev-parse --symbolic "
3141         cmdline += " --remotes"
3142
3143         for line in read_pipe_lines(cmdline):
3144             line = line.strip()
3145
3146             if not line.startswith('p4/') or line == "p4/HEAD":
3147                 continue
3148             branch = line
3149
3150             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3151             settings = extractSettingsGitLog(log)
3152
3153             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3154         return True
3155
3156 class HelpFormatter(optparse.IndentedHelpFormatter):
3157     def __init__(self):
3158         optparse.IndentedHelpFormatter.__init__(self)
3159
3160     def format_description(self, description):
3161         if description:
3162             return description + "\n"
3163         else:
3164             return ""
3165
3166 def printUsage(commands):
3167     print "usage: %s <command> [options]" % sys.argv[0]
3168     print ""
3169     print "valid commands: %s" % ", ".join(commands)
3170     print ""
3171     print "Try %s <command> --help for command specific help." % sys.argv[0]
3172     print ""
3173
3174 commands = {
3175     "debug" : P4Debug,
3176     "submit" : P4Submit,
3177     "commit" : P4Submit,
3178     "sync" : P4Sync,
3179     "rebase" : P4Rebase,
3180     "clone" : P4Clone,
3181     "rollback" : P4RollBack,
3182     "branches" : P4Branches
3183 }
3184
3185
3186 def main():
3187     if len(sys.argv[1:]) == 0:
3188         printUsage(commands.keys())
3189         sys.exit(2)
3190
3191     cmdName = sys.argv[1]
3192     try:
3193         klass = commands[cmdName]
3194         cmd = klass()
3195     except KeyError:
3196         print "unknown command %s" % cmdName
3197         print ""
3198         printUsage(commands.keys())
3199         sys.exit(2)
3200
3201     options = cmd.options
3202     cmd.gitdir = os.environ.get("GIT_DIR", None)
3203
3204     args = sys.argv[2:]
3205
3206     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3207     if cmd.needsGit:
3208         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3209
3210     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3211                                    options,
3212                                    description = cmd.description,
3213                                    formatter = HelpFormatter())
3214
3215     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3216     global verbose
3217     verbose = cmd.verbose
3218     if cmd.needsGit:
3219         if cmd.gitdir == None:
3220             cmd.gitdir = os.path.abspath(".git")
3221             if not isValidGitDir(cmd.gitdir):
3222                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3223                 if os.path.exists(cmd.gitdir):
3224                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3225                     if len(cdup) > 0:
3226                         chdir(cdup);
3227
3228         if not isValidGitDir(cmd.gitdir):
3229             if isValidGitDir(cmd.gitdir + "/.git"):
3230                 cmd.gitdir += "/.git"
3231             else:
3232                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3233
3234         os.environ["GIT_DIR"] = cmd.gitdir
3235
3236     if not cmd.run(args):
3237         parser.print_help()
3238         sys.exit(2)
3239
3240
3241 if __name__ == '__main__':
3242     main()