git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import sys
  12 if sys.hexversion < 0x02040000:
  13     # The limiter is the subprocess module
  14     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  15     sys.exit(1)
  16
  17 import optparse, os, marshal, subprocess, shelve
  18 import tempfile, getopt, os.path, time, platform
  19 import re, shutil
  20
  21 verbose = False
  22
  23 # Only labels/tags matching this will be imported/exported
  24 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  25
  26 def p4_build_cmd(cmd):
  27     """Build a suitable p4 command line.
  28
  29     This consolidates building and returning a p4 command line into one
  30     location. It means that hooking into the environment, or other configuration
  31     can be done more easily.
  32     """
  33     real_cmd = ["p4"]
  34
  35     user = gitConfig("git-p4.user")
  36     if len(user) > 0:
  37         real_cmd += ["-u",user]
  38
  39     password = gitConfig("git-p4.password")
  40     if len(password) > 0:
  41         real_cmd += ["-P", password]
  42
  43     port = gitConfig("git-p4.port")
  44     if len(port) > 0:
  45         real_cmd += ["-p", port]
  46
  47     host = gitConfig("git-p4.host")
  48     if len(host) > 0:
  49         real_cmd += ["-H", host]
  50
  51     client = gitConfig("git-p4.client")
  52     if len(client) > 0:
  53         real_cmd += ["-c", client]
  54
  55
  56     if isinstance(cmd,basestring):
  57         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  58     else:
  59         real_cmd += cmd
  60     return real_cmd
  61
  62 def chdir(dir):
  63     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  64     # not using the shell, we have to set it ourselves.  This path could
  65     # be relative, so go there first, then figure out where we ended up.
  66     os.chdir(dir)
  67     os.environ['PWD'] = os.getcwd()
  68
  69 def die(msg):
  70     if verbose:
  71         raise Exception(msg)
  72     else:
  73         sys.stderr.write(msg + "\n")
  74         sys.exit(1)
  75
  76 def write_pipe(c, stdin):
  77     if verbose:
  78         sys.stderr.write('Writing pipe: %s\n' % str(c))
  79
  80     expand = isinstance(c,basestring)
  81     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  82     pipe = p.stdin
  83     val = pipe.write(stdin)
  84     pipe.close()
  85     if p.wait():
  86         die('Command failed: %s' % str(c))
  87
  88     return val
  89
  90 def p4_write_pipe(c, stdin):
  91     real_cmd = p4_build_cmd(c)
  92     return write_pipe(real_cmd, stdin)
  93
  94 def read_pipe(c, ignore_error=False):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % str(c))
  97
  98     expand = isinstance(c,basestring)
  99     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 100     pipe = p.stdout
 101     val = pipe.read()
 102     if p.wait() and not ignore_error:
 103         die('Command failed: %s' % str(c))
 104
 105     return val
 106
 107 def p4_read_pipe(c, ignore_error=False):
 108     real_cmd = p4_build_cmd(c)
 109     return read_pipe(real_cmd, ignore_error)
 110
 111 def read_pipe_lines(c):
 112     if verbose:
 113         sys.stderr.write('Reading pipe: %s\n' % str(c))
 114
 115     expand = isinstance(c, basestring)
 116     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 117     pipe = p.stdout
 118     val = pipe.readlines()
 119     if pipe.close() or p.wait():
 120         die('Command failed: %s' % str(c))
 121
 122     return val
 123
 124 def p4_read_pipe_lines(c):
 125     """Specifically invoke p4 on the command supplied. """
 126     real_cmd = p4_build_cmd(c)
 127     return read_pipe_lines(real_cmd)
 128
 129 def p4_has_command(cmd):
 130     """Ask p4 for help on this command.  If it returns an error, the
 131        command does not exist in this version of p4."""
 132     real_cmd = p4_build_cmd(["help", cmd])
 133     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 134                                    stderr=subprocess.PIPE)
 135     p.communicate()
 136     return p.returncode == 0
 137
 138 def p4_has_move_command():
 139     """See if the move command exists, that it supports -k, and that
 140        it has not been administratively disabled.  The arguments
 141        must be correct, but the filenames do not have to exist.  Use
 142        ones with wildcards so even if they exist, it will fail."""
 143
 144     if not p4_has_command("move"):
 145         return False
 146     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 147     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 148     (out, err) = p.communicate()
 149     # return code will be 1 in either case
 150     if err.find("Invalid option") >= 0:
 151         return False
 152     if err.find("disabled") >= 0:
 153         return False
 154     # assume it failed because @... was invalid changelist
 155     return True
 156
 157 def system(cmd):
 158     expand = isinstance(cmd,basestring)
 159     if verbose:
 160         sys.stderr.write("executing %s\n" % str(cmd))
 161     subprocess.check_call(cmd, shell=expand)
 162
 163 def p4_system(cmd):
 164     """Specifically invoke p4 as the system command. """
 165     real_cmd = p4_build_cmd(cmd)
 166     expand = isinstance(real_cmd, basestring)
 167     subprocess.check_call(real_cmd, shell=expand)
 168
 169 def p4_integrate(src, dest):
 170     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 171
 172 def p4_sync(f, *options):
 173     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 174
 175 def p4_add(f):
 176     # forcibly add file names with wildcards
 177     if wildcard_present(f):
 178         p4_system(["add", "-f", f])
 179     else:
 180         p4_system(["add", f])
 181
 182 def p4_delete(f):
 183     p4_system(["delete", wildcard_encode(f)])
 184
 185 def p4_edit(f):
 186     p4_system(["edit", wildcard_encode(f)])
 187
 188 def p4_revert(f):
 189     p4_system(["revert", wildcard_encode(f)])
 190
 191 def p4_reopen(type, f):
 192     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 193
 194 def p4_move(src, dest):
 195     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 196
 197 def p4_describe(change):
 198     """Make sure it returns a valid result by checking for
 199        the presence of field "time".  Return a dict of the
 200        results."""
 201
 202     ds = p4CmdList(["describe", "-s", str(change)])
 203     if len(ds) != 1:
 204         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 205
 206     d = ds[0]
 207
 208     if "p4ExitCode" in d:
 209         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 210                                                       str(d)))
 211     if "code" in d:
 212         if d["code"] == "error":
 213             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 214
 215     if "time" not in d:
 216         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 217
 218     return d
 219
 220 #
 221 # Canonicalize the p4 type and return a tuple of the
 222 # base type, plus any modifiers.  See "p4 help filetypes"
 223 # for a list and explanation.
 224 #
 225 def split_p4_type(p4type):
 226
 227     p4_filetypes_historical = {
 228         "ctempobj": "binary+Sw",
 229         "ctext": "text+C",
 230         "cxtext": "text+Cx",
 231         "ktext": "text+k",
 232         "kxtext": "text+kx",
 233         "ltext": "text+F",
 234         "tempobj": "binary+FSw",
 235         "ubinary": "binary+F",
 236         "uresource": "resource+F",
 237         "uxbinary": "binary+Fx",
 238         "xbinary": "binary+x",
 239         "xltext": "text+Fx",
 240         "xtempobj": "binary+Swx",
 241         "xtext": "text+x",
 242         "xunicode": "unicode+x",
 243         "xutf16": "utf16+x",
 244     }
 245     if p4type in p4_filetypes_historical:
 246         p4type = p4_filetypes_historical[p4type]
 247     mods = ""
 248     s = p4type.split("+")
 249     base = s[0]
 250     mods = ""
 251     if len(s) > 1:
 252         mods = s[1]
 253     return (base, mods)
 254
 255 #
 256 # return the raw p4 type of a file (text, text+ko, etc)
 257 #
 258 def p4_type(file):
 259     results = p4CmdList(["fstat", "-T", "headType", file])
 260     return results[0]['headType']
 261
 262 #
 263 # Given a type base and modifier, return a regexp matching
 264 # the keywords that can be expanded in the file
 265 #
 266 def p4_keywords_regexp_for_type(base, type_mods):
 267     if base in ("text", "unicode", "binary"):
 268         kwords = None
 269         if "ko" in type_mods:
 270             kwords = 'Id|Header'
 271         elif "k" in type_mods:
 272             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 273         else:
 274             return None
 275         pattern = r"""
 276             \$              # Starts with a dollar, followed by...
 277             (%s)            # one of the keywords, followed by...
 278             (:[^$\n]+)?     # possibly an old expansion, followed by...
 279             \$              # another dollar
 280             """ % kwords
 281         return pattern
 282     else:
 283         return None
 284
 285 #
 286 # Given a file, return a regexp matching the possible
 287 # RCS keywords that will be expanded, or None for files
 288 # with kw expansion turned off.
 289 #
 290 def p4_keywords_regexp_for_file(file):
 291     if not os.path.exists(file):
 292         return None
 293     else:
 294         (type_base, type_mods) = split_p4_type(p4_type(file))
 295         return p4_keywords_regexp_for_type(type_base, type_mods)
 296
 297 def setP4ExecBit(file, mode):
 298     # Reopens an already open file and changes the execute bit to match
 299     # the execute bit setting in the passed in mode.
 300
 301     p4Type = "+x"
 302
 303     if not isModeExec(mode):
 304         p4Type = getP4OpenedType(file)
 305         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 306         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 307         if p4Type[-1] == "+":
 308             p4Type = p4Type[0:-1]
 309
 310     p4_reopen(p4Type, file)
 311
 312 def getP4OpenedType(file):
 313     # Returns the perforce file type for the given file.
 314
 315     result = p4_read_pipe(["opened", wildcard_encode(file)])
 316     match = re.match(".*\((.+)\)\r?$", result)
 317     if match:
 318         return match.group(1)
 319     else:
 320         die("Could not determine file type for %s (result: '%s')" % (file, result))
 321
 322 # Return the set of all p4 labels
 323 def getP4Labels(depotPaths):
 324     labels = set()
 325     if isinstance(depotPaths,basestring):
 326         depotPaths = [depotPaths]
 327
 328     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 329         label = l['label']
 330         labels.add(label)
 331
 332     return labels
 333
 334 # Return the set of all git tags
 335 def getGitTags():
 336     gitTags = set()
 337     for line in read_pipe_lines(["git", "tag"]):
 338         tag = line.strip()
 339         gitTags.add(tag)
 340     return gitTags
 341
 342 def diffTreePattern():
 343     # This is a simple generator for the diff tree regex pattern. This could be
 344     # a class variable if this and parseDiffTreeEntry were a part of a class.
 345     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 346     while True:
 347         yield pattern
 348
 349 def parseDiffTreeEntry(entry):
 350     """Parses a single diff tree entry into its component elements.
 351
 352     See git-diff-tree(1) manpage for details about the format of the diff
 353     output. This method returns a dictionary with the following elements:
 354
 355     src_mode - The mode of the source file
 356     dst_mode - The mode of the destination file
 357     src_sha1 - The sha1 for the source file
 358     dst_sha1 - The sha1 fr the destination file
 359     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 360     status_score - The score for the status (applicable for 'C' and 'R'
 361                    statuses). This is None if there is no score.
 362     src - The path for the source file.
 363     dst - The path for the destination file. This is only present for
 364           copy or renames. If it is not present, this is None.
 365
 366     If the pattern is not matched, None is returned."""
 367
 368     match = diffTreePattern().next().match(entry)
 369     if match:
 370         return {
 371             'src_mode': match.group(1),
 372             'dst_mode': match.group(2),
 373             'src_sha1': match.group(3),
 374             'dst_sha1': match.group(4),
 375             'status': match.group(5),
 376             'status_score': match.group(6),
 377             'src': match.group(7),
 378             'dst': match.group(10)
 379         }
 380     return None
 381
 382 def isModeExec(mode):
 383     # Returns True if the given git mode represents an executable file,
 384     # otherwise False.
 385     return mode[-3:] == "755"
 386
 387 def isModeExecChanged(src_mode, dst_mode):
 388     return isModeExec(src_mode) != isModeExec(dst_mode)
 389
 390 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 391
 392     if isinstance(cmd,basestring):
 393         cmd = "-G " + cmd
 394         expand = True
 395     else:
 396         cmd = ["-G"] + cmd
 397         expand = False
 398
 399     cmd = p4_build_cmd(cmd)
 400     if verbose:
 401         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 402
 403     # Use a temporary file to avoid deadlocks without
 404     # subprocess.communicate(), which would put another copy
 405     # of stdout into memory.
 406     stdin_file = None
 407     if stdin is not None:
 408         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 409         if isinstance(stdin,basestring):
 410             stdin_file.write(stdin)
 411         else:
 412             for i in stdin:
 413                 stdin_file.write(i + '\n')
 414         stdin_file.flush()
 415         stdin_file.seek(0)
 416
 417     p4 = subprocess.Popen(cmd,
 418                           shell=expand,
 419                           stdin=stdin_file,
 420                           stdout=subprocess.PIPE)
 421
 422     result = []
 423     try:
 424         while True:
 425             entry = marshal.load(p4.stdout)
 426             if cb is not None:
 427                 cb(entry)
 428             else:
 429                 result.append(entry)
 430     except EOFError:
 431         pass
 432     exitCode = p4.wait()
 433     if exitCode != 0:
 434         entry = {}
 435         entry["p4ExitCode"] = exitCode
 436         result.append(entry)
 437
 438     return result
 439
 440 def p4Cmd(cmd):
 441     list = p4CmdList(cmd)
 442     result = {}
 443     for entry in list:
 444         result.update(entry)
 445     return result;
 446
 447 def p4Where(depotPath):
 448     if not depotPath.endswith("/"):
 449         depotPath += "/"
 450     depotPath = depotPath + "..."
 451     outputList = p4CmdList(["where", depotPath])
 452     output = None
 453     for entry in outputList:
 454         if "depotFile" in entry:
 455             if entry["depotFile"] == depotPath:
 456                 output = entry
 457                 break
 458         elif "data" in entry:
 459             data = entry.get("data")
 460             space = data.find(" ")
 461             if data[:space] == depotPath:
 462                 output = entry
 463                 break
 464     if output == None:
 465         return ""
 466     if output["code"] == "error":
 467         return ""
 468     clientPath = ""
 469     if "path" in output:
 470         clientPath = output.get("path")
 471     elif "data" in output:
 472         data = output.get("data")
 473         lastSpace = data.rfind(" ")
 474         clientPath = data[lastSpace + 1:]
 475
 476     if clientPath.endswith("..."):
 477         clientPath = clientPath[:-3]
 478     return clientPath
 479
 480 def currentGitBranch():
 481     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 482
 483 def isValidGitDir(path):
 484     if (os.path.exists(path + "/HEAD")
 485         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 486         return True;
 487     return False
 488
 489 def parseRevision(ref):
 490     return read_pipe("git rev-parse %s" % ref).strip()
 491
 492 def branchExists(ref):
 493     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 494                      ignore_error=True)
 495     return len(rev) > 0
 496
 497 def extractLogMessageFromGitCommit(commit):
 498     logMessage = ""
 499
 500     ## fixme: title is first line of commit, not 1st paragraph.
 501     foundTitle = False
 502     for log in read_pipe_lines("git cat-file commit %s" % commit):
 503        if not foundTitle:
 504            if len(log) == 1:
 505                foundTitle = True
 506            continue
 507
 508        logMessage += log
 509     return logMessage
 510
 511 def extractSettingsGitLog(log):
 512     values = {}
 513     for line in log.split("\n"):
 514         line = line.strip()
 515         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 516         if not m:
 517             continue
 518
 519         assignments = m.group(1).split (':')
 520         for a in assignments:
 521             vals = a.split ('=')
 522             key = vals[0].strip()
 523             val = ('='.join (vals[1:])).strip()
 524             if val.endswith ('\"') and val.startswith('"'):
 525                 val = val[1:-1]
 526
 527             values[key] = val
 528
 529     paths = values.get("depot-paths")
 530     if not paths:
 531         paths = values.get("depot-path")
 532     if paths:
 533         values['depot-paths'] = paths.split(',')
 534     return values
 535
 536 def gitBranchExists(branch):
 537     proc = subprocess.Popen(["git", "rev-parse", branch],
 538                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 539     return proc.wait() == 0;
 540
 541 _gitConfig = {}
 542 def gitConfig(key, args = None): # set args to "--bool", for instance
 543     if not _gitConfig.has_key(key):
 544         argsFilter = ""
 545         if args != None:
 546             argsFilter = "%s " % args
 547         cmd = "git config %s%s" % (argsFilter, key)
 548         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 549     return _gitConfig[key]
 550
 551 def gitConfigList(key):
 552     if not _gitConfig.has_key(key):
 553         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 554     return _gitConfig[key]
 555
 556 def p4BranchesInGit(branchesAreInRemotes=True):
 557     """Find all the branches whose names start with "p4/", looking
 558        in remotes or heads as specified by the argument.  Return
 559        a dictionary of { branch: revision } for each one found.
 560        The branch names are the short names, without any
 561        "p4/" prefix."""
 562
 563     branches = {}
 564
 565     cmdline = "git rev-parse --symbolic "
 566     if branchesAreInRemotes:
 567         cmdline += "--remotes"
 568     else:
 569         cmdline += "--branches"
 570
 571     for line in read_pipe_lines(cmdline):
 572         line = line.strip()
 573
 574         # only import to p4/
 575         if not line.startswith('p4/'):
 576             continue
 577         # special symbolic ref to p4/master
 578         if line == "p4/HEAD":
 579             continue
 580
 581         # strip off p4/ prefix
 582         branch = line[len("p4/"):]
 583
 584         branches[branch] = parseRevision(line)
 585
 586     return branches
 587
 588 def branch_exists(branch):
 589     """Make sure that the given ref name really exists."""
 590
 591     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 592     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 593     out, _ = p.communicate()
 594     if p.returncode:
 595         return False
 596     # expect exactly one line of output: the branch name
 597     return out.rstrip() == branch
 598
 599 def findUpstreamBranchPoint(head = "HEAD"):
 600     branches = p4BranchesInGit()
 601     # map from depot-path to branch name
 602     branchByDepotPath = {}
 603     for branch in branches.keys():
 604         tip = branches[branch]
 605         log = extractLogMessageFromGitCommit(tip)
 606         settings = extractSettingsGitLog(log)
 607         if settings.has_key("depot-paths"):
 608             paths = ",".join(settings["depot-paths"])
 609             branchByDepotPath[paths] = "remotes/p4/" + branch
 610
 611     settings = None
 612     parent = 0
 613     while parent < 65535:
 614         commit = head + "~%s" % parent
 615         log = extractLogMessageFromGitCommit(commit)
 616         settings = extractSettingsGitLog(log)
 617         if settings.has_key("depot-paths"):
 618             paths = ",".join(settings["depot-paths"])
 619             if branchByDepotPath.has_key(paths):
 620                 return [branchByDepotPath[paths], settings]
 621
 622         parent = parent + 1
 623
 624     return ["", settings]
 625
 626 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 627     if not silent:
 628         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 629                % localRefPrefix)
 630
 631     originPrefix = "origin/p4/"
 632
 633     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 634         line = line.strip()
 635         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 636             continue
 637
 638         headName = line[len(originPrefix):]
 639         remoteHead = localRefPrefix + headName
 640         originHead = line
 641
 642         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 643         if (not original.has_key('depot-paths')
 644             or not original.has_key('change')):
 645             continue
 646
 647         update = False
 648         if not gitBranchExists(remoteHead):
 649             if verbose:
 650                 print "creating %s" % remoteHead
 651             update = True
 652         else:
 653             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 654             if settings.has_key('change') > 0:
 655                 if settings['depot-paths'] == original['depot-paths']:
 656                     originP4Change = int(original['change'])
 657                     p4Change = int(settings['change'])
 658                     if originP4Change > p4Change:
 659                         print ("%s (%s) is newer than %s (%s). "
 660                                "Updating p4 branch from origin."
 661                                % (originHead, originP4Change,
 662                                   remoteHead, p4Change))
 663                         update = True
 664                 else:
 665                     print ("Ignoring: %s was imported from %s while "
 666                            "%s was imported from %s"
 667                            % (originHead, ','.join(original['depot-paths']),
 668                               remoteHead, ','.join(settings['depot-paths'])))
 669
 670         if update:
 671             system("git update-ref %s %s" % (remoteHead, originHead))
 672
 673 def originP4BranchesExist():
 674         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 675
 676 def p4ChangesForPaths(depotPaths, changeRange):
 677     assert depotPaths
 678     cmd = ['changes']
 679     for p in depotPaths:
 680         cmd += ["%s...%s" % (p, changeRange)]
 681     output = p4_read_pipe_lines(cmd)
 682
 683     changes = {}
 684     for line in output:
 685         changeNum = int(line.split(" ")[1])
 686         changes[changeNum] = True
 687
 688     changelist = changes.keys()
 689     changelist.sort()
 690     return changelist
 691
 692 def p4PathStartsWith(path, prefix):
 693     # This method tries to remedy a potential mixed-case issue:
 694     #
 695     # If UserA adds  //depot/DirA/file1
 696     # and UserB adds //depot/dira/file2
 697     #
 698     # we may or may not have a problem. If you have core.ignorecase=true,
 699     # we treat DirA and dira as the same directory
 700     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 701     if ignorecase:
 702         return path.lower().startswith(prefix.lower())
 703     return path.startswith(prefix)
 704
 705 def getClientSpec():
 706     """Look at the p4 client spec, create a View() object that contains
 707        all the mappings, and return it."""
 708
 709     specList = p4CmdList("client -o")
 710     if len(specList) != 1:
 711         die('Output from "client -o" is %d lines, expecting 1' %
 712             len(specList))
 713
 714     # dictionary of all client parameters
 715     entry = specList[0]
 716
 717     # just the keys that start with "View"
 718     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 719
 720     # hold this new View
 721     view = View()
 722
 723     # append the lines, in order, to the view
 724     for view_num in range(len(view_keys)):
 725         k = "View%d" % view_num
 726         if k not in view_keys:
 727             die("Expected view key %s missing" % k)
 728         view.append(entry[k])
 729
 730     return view
 731
 732 def getClientRoot():
 733     """Grab the client directory."""
 734
 735     output = p4CmdList("client -o")
 736     if len(output) != 1:
 737         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 738
 739     entry = output[0]
 740     if "Root" not in entry:
 741         die('Client has no "Root"')
 742
 743     return entry["Root"]
 744
 745 #
 746 # P4 wildcards are not allowed in filenames.  P4 complains
 747 # if you simply add them, but you can force it with "-f", in
 748 # which case it translates them into %xx encoding internally.
 749 #
 750 def wildcard_decode(path):
 751     # Search for and fix just these four characters.  Do % last so
 752     # that fixing it does not inadvertently create new %-escapes.
 753     # Cannot have * in a filename in windows; untested as to
 754     # what p4 would do in such a case.
 755     if not platform.system() == "Windows":
 756         path = path.replace("%2A", "*")
 757     path = path.replace("%23", "#") \
 758                .replace("%40", "@") \
 759                .replace("%25", "%")
 760     return path
 761
 762 def wildcard_encode(path):
 763     # do % first to avoid double-encoding the %s introduced here
 764     path = path.replace("%", "%25") \
 765                .replace("*", "%2A") \
 766                .replace("#", "%23") \
 767                .replace("@", "%40")
 768     return path
 769
 770 def wildcard_present(path):
 771     return path.translate(None, "*#@%") != path
 772
 773 class Command:
 774     def __init__(self):
 775         self.usage = "usage: %prog [options]"
 776         self.needsGit = True
 777         self.verbose = False
 778
 779 class P4UserMap:
 780     def __init__(self):
 781         self.userMapFromPerforceServer = False
 782         self.myP4UserId = None
 783
 784     def p4UserId(self):
 785         if self.myP4UserId:
 786             return self.myP4UserId
 787
 788         results = p4CmdList("user -o")
 789         for r in results:
 790             if r.has_key('User'):
 791                 self.myP4UserId = r['User']
 792                 return r['User']
 793         die("Could not find your p4 user id")
 794
 795     def p4UserIsMe(self, p4User):
 796         # return True if the given p4 user is actually me
 797         me = self.p4UserId()
 798         if not p4User or p4User != me:
 799             return False
 800         else:
 801             return True
 802
 803     def getUserCacheFilename(self):
 804         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 805         return home + "/.gitp4-usercache.txt"
 806
 807     def getUserMapFromPerforceServer(self):
 808         if self.userMapFromPerforceServer:
 809             return
 810         self.users = {}
 811         self.emails = {}
 812
 813         for output in p4CmdList("users"):
 814             if not output.has_key("User"):
 815                 continue
 816             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 817             self.emails[output["Email"]] = output["User"]
 818
 819
 820         s = ''
 821         for (key, val) in self.users.items():
 822             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 823
 824         open(self.getUserCacheFilename(), "wb").write(s)
 825         self.userMapFromPerforceServer = True
 826
 827     def loadUserMapFromCache(self):
 828         self.users = {}
 829         self.userMapFromPerforceServer = False
 830         try:
 831             cache = open(self.getUserCacheFilename(), "rb")
 832             lines = cache.readlines()
 833             cache.close()
 834             for line in lines:
 835                 entry = line.strip().split("\t")
 836                 self.users[entry[0]] = entry[1]
 837         except IOError:
 838             self.getUserMapFromPerforceServer()
 839
 840 class P4Debug(Command):
 841     def __init__(self):
 842         Command.__init__(self)
 843         self.options = []
 844         self.description = "A tool to debug the output of p4 -G."
 845         self.needsGit = False
 846
 847     def run(self, args):
 848         j = 0
 849         for output in p4CmdList(args):
 850             print 'Element: %d' % j
 851             j += 1
 852             print output
 853         return True
 854
 855 class P4RollBack(Command):
 856     def __init__(self):
 857         Command.__init__(self)
 858         self.options = [
 859             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 860         ]
 861         self.description = "A tool to debug the multi-branch import. Don't use :)"
 862         self.rollbackLocalBranches = False
 863
 864     def run(self, args):
 865         if len(args) != 1:
 866             return False
 867         maxChange = int(args[0])
 868
 869         if "p4ExitCode" in p4Cmd("changes -m 1"):
 870             die("Problems executing p4");
 871
 872         if self.rollbackLocalBranches:
 873             refPrefix = "refs/heads/"
 874             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 875         else:
 876             refPrefix = "refs/remotes/"
 877             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 878
 879         for line in lines:
 880             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 881                 line = line.strip()
 882                 ref = refPrefix + line
 883                 log = extractLogMessageFromGitCommit(ref)
 884                 settings = extractSettingsGitLog(log)
 885
 886                 depotPaths = settings['depot-paths']
 887                 change = settings['change']
 888
 889                 changed = False
 890
 891                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 892                                                            for p in depotPaths]))) == 0:
 893                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 894                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 895                     continue
 896
 897                 while change and int(change) > maxChange:
 898                     changed = True
 899                     if self.verbose:
 900                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 901                     system("git update-ref %s \"%s^\"" % (ref, ref))
 902                     log = extractLogMessageFromGitCommit(ref)
 903                     settings =  extractSettingsGitLog(log)
 904
 905
 906                     depotPaths = settings['depot-paths']
 907                     change = settings['change']
 908
 909                 if changed:
 910                     print "%s rewound to %s" % (ref, change)
 911
 912         return True
 913
 914 class P4Submit(Command, P4UserMap):
 915
 916     conflict_behavior_choices = ("ask", "skip", "quit")
 917
 918     def __init__(self):
 919         Command.__init__(self)
 920         P4UserMap.__init__(self)
 921         self.options = [
 922                 optparse.make_option("--origin", dest="origin"),
 923                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 924                 # preserve the user, requires relevant p4 permissions
 925                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 926                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 927                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
 928                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
 929                 optparse.make_option("--conflict", dest="conflict_behavior",
 930                                      choices=self.conflict_behavior_choices),
 931                 optparse.make_option("--branch", dest="branch"),
 932         ]
 933         self.description = "Submit changes from git to the perforce depot."
 934         self.usage += " [name of git branch to submit into perforce depot]"
 935         self.origin = ""
 936         self.detectRenames = False
 937         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 938         self.dry_run = False
 939         self.prepare_p4_only = False
 940         self.conflict_behavior = None
 941         self.isWindows = (platform.system() == "Windows")
 942         self.exportLabels = False
 943         self.p4HasMoveCommand = p4_has_move_command()
 944         self.branch = None
 945
 946     def check(self):
 947         if len(p4CmdList("opened ...")) > 0:
 948             die("You have files opened with perforce! Close them before starting the sync.")
 949
 950     def separate_jobs_from_description(self, message):
 951         """Extract and return a possible Jobs field in the commit
 952            message.  It goes into a separate section in the p4 change
 953            specification.
 954
 955            A jobs line starts with "Jobs:" and looks like a new field
 956            in a form.  Values are white-space separated on the same
 957            line or on following lines that start with a tab.
 958
 959            This does not parse and extract the full git commit message
 960            like a p4 form.  It just sees the Jobs: line as a marker
 961            to pass everything from then on directly into the p4 form,
 962            but outside the description section.
 963
 964            Return a tuple (stripped log message, jobs string)."""
 965
 966         m = re.search(r'^Jobs:', message, re.MULTILINE)
 967         if m is None:
 968             return (message, None)
 969
 970         jobtext = message[m.start():]
 971         stripped_message = message[:m.start()].rstrip()
 972         return (stripped_message, jobtext)
 973
 974     def prepareLogMessage(self, template, message, jobs):
 975         """Edits the template returned from "p4 change -o" to insert
 976            the message in the Description field, and the jobs text in
 977            the Jobs field."""
 978         result = ""
 979
 980         inDescriptionSection = False
 981
 982         for line in template.split("\n"):
 983             if line.startswith("#"):
 984                 result += line + "\n"
 985                 continue
 986
 987             if inDescriptionSection:
 988                 if line.startswith("Files:") or line.startswith("Jobs:"):
 989                     inDescriptionSection = False
 990                     # insert Jobs section
 991                     if jobs:
 992                         result += jobs + "\n"
 993                 else:
 994                     continue
 995             else:
 996                 if line.startswith("Description:"):
 997                     inDescriptionSection = True
 998                     line += "\n"
 999                     for messageLine in message.split("\n"):
1000                         line += "\t" + messageLine + "\n"
1001
1002             result += line + "\n"
1003
1004         return result
1005
1006     def patchRCSKeywords(self, file, pattern):
1007         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1008         (handle, outFileName) = tempfile.mkstemp(dir='.')
1009         try:
1010             outFile = os.fdopen(handle, "w+")
1011             inFile = open(file, "r")
1012             regexp = re.compile(pattern, re.VERBOSE)
1013             for line in inFile.readlines():
1014                 line = regexp.sub(r'$\1$', line)
1015                 outFile.write(line)
1016             inFile.close()
1017             outFile.close()
1018             # Forcibly overwrite the original file
1019             os.unlink(file)
1020             shutil.move(outFileName, file)
1021         except:
1022             # cleanup our temporary file
1023             os.unlink(outFileName)
1024             print "Failed to strip RCS keywords in %s" % file
1025             raise
1026
1027         print "Patched up RCS keywords in %s" % file
1028
1029     def p4UserForCommit(self,id):
1030         # Return the tuple (perforce user,git email) for a given git commit id
1031         self.getUserMapFromPerforceServer()
1032         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
1033         gitEmail = gitEmail.strip()
1034         if not self.emails.has_key(gitEmail):
1035             return (None,gitEmail)
1036         else:
1037             return (self.emails[gitEmail],gitEmail)
1038
1039     def checkValidP4Users(self,commits):
1040         # check if any git authors cannot be mapped to p4 users
1041         for id in commits:
1042             (user,email) = self.p4UserForCommit(id)
1043             if not user:
1044                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1045                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
1046                     print "%s" % msg
1047                 else:
1048                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1049
1050     def lastP4Changelist(self):
1051         # Get back the last changelist number submitted in this client spec. This
1052         # then gets used to patch up the username in the change. If the same
1053         # client spec is being used by multiple processes then this might go
1054         # wrong.
1055         results = p4CmdList("client -o")        # find the current client
1056         client = None
1057         for r in results:
1058             if r.has_key('Client'):
1059                 client = r['Client']
1060                 break
1061         if not client:
1062             die("could not get client spec")
1063         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1064         for r in results:
1065             if r.has_key('change'):
1066                 return r['change']
1067         die("Could not get changelist number for last submit - cannot patch up user details")
1068
1069     def modifyChangelistUser(self, changelist, newUser):
1070         # fixup the user field of a changelist after it has been submitted.
1071         changes = p4CmdList("change -o %s" % changelist)
1072         if len(changes) != 1:
1073             die("Bad output from p4 change modifying %s to user %s" %
1074                 (changelist, newUser))
1075
1076         c = changes[0]
1077         if c['User'] == newUser: return   # nothing to do
1078         c['User'] = newUser
1079         input = marshal.dumps(c)
1080
1081         result = p4CmdList("change -f -i", stdin=input)
1082         for r in result:
1083             if r.has_key('code'):
1084                 if r['code'] == 'error':
1085                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1086             if r.has_key('data'):
1087                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1088                 return
1089         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1090
1091     def canChangeChangelists(self):
1092         # check to see if we have p4 admin or super-user permissions, either of
1093         # which are required to modify changelists.
1094         results = p4CmdList(["protects", self.depotPath])
1095         for r in results:
1096             if r.has_key('perm'):
1097                 if r['perm'] == 'admin':
1098                     return 1
1099                 if r['perm'] == 'super':
1100                     return 1
1101         return 0
1102
1103     def prepareSubmitTemplate(self):
1104         """Run "p4 change -o" to grab a change specification template.
1105            This does not use "p4 -G", as it is nice to keep the submission
1106            template in original order, since a human might edit it.
1107
1108            Remove lines in the Files section that show changes to files
1109            outside the depot path we're committing into."""
1110
1111         template = ""
1112         inFilesSection = False
1113         for line in p4_read_pipe_lines(['change', '-o']):
1114             if line.endswith("\r\n"):
1115                 line = line[:-2] + "\n"
1116             if inFilesSection:
1117                 if line.startswith("\t"):
1118                     # path starts and ends with a tab
1119                     path = line[1:]
1120                     lastTab = path.rfind("\t")
1121                     if lastTab != -1:
1122                         path = path[:lastTab]
1123                         if not p4PathStartsWith(path, self.depotPath):
1124                             continue
1125                 else:
1126                     inFilesSection = False
1127             else:
1128                 if line.startswith("Files:"):
1129                     inFilesSection = True
1130
1131             template += line
1132
1133         return template
1134
1135     def edit_template(self, template_file):
1136         """Invoke the editor to let the user change the submission
1137            message.  Return true if okay to continue with the submit."""
1138
1139         # if configured to skip the editing part, just submit
1140         if gitConfig("git-p4.skipSubmitEdit") == "true":
1141             return True
1142
1143         # look at the modification time, to check later if the user saved
1144         # the file
1145         mtime = os.stat(template_file).st_mtime
1146
1147         # invoke the editor
1148         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1149             editor = os.environ.get("P4EDITOR")
1150         else:
1151             editor = read_pipe("git var GIT_EDITOR").strip()
1152         system(editor + " " + template_file)
1153
1154         # If the file was not saved, prompt to see if this patch should
1155         # be skipped.  But skip this verification step if configured so.
1156         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1157             return True
1158
1159         # modification time updated means user saved the file
1160         if os.stat(template_file).st_mtime > mtime:
1161             return True
1162
1163         while True:
1164             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1165             if response == 'y':
1166                 return True
1167             if response == 'n':
1168                 return False
1169
1170     def applyCommit(self, id):
1171         """Apply one commit, return True if it succeeded."""
1172
1173         print "Applying", read_pipe(["git", "show", "-s",
1174                                      "--format=format:%h %s", id])
1175
1176         (p4User, gitEmail) = self.p4UserForCommit(id)
1177
1178         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1179         filesToAdd = set()
1180         filesToDelete = set()
1181         editedFiles = set()
1182         pureRenameCopy = set()
1183         filesToChangeExecBit = {}
1184
1185         for line in diff:
1186             diff = parseDiffTreeEntry(line)
1187             modifier = diff['status']
1188             path = diff['src']
1189             if modifier == "M":
1190                 p4_edit(path)
1191                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1192                     filesToChangeExecBit[path] = diff['dst_mode']
1193                 editedFiles.add(path)
1194             elif modifier == "A":
1195                 filesToAdd.add(path)
1196                 filesToChangeExecBit[path] = diff['dst_mode']
1197                 if path in filesToDelete:
1198                     filesToDelete.remove(path)
1199             elif modifier == "D":
1200                 filesToDelete.add(path)
1201                 if path in filesToAdd:
1202                     filesToAdd.remove(path)
1203             elif modifier == "C":
1204                 src, dest = diff['src'], diff['dst']
1205                 p4_integrate(src, dest)
1206                 pureRenameCopy.add(dest)
1207                 if diff['src_sha1'] != diff['dst_sha1']:
1208                     p4_edit(dest)
1209                     pureRenameCopy.discard(dest)
1210                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1211                     p4_edit(dest)
1212                     pureRenameCopy.discard(dest)
1213                     filesToChangeExecBit[dest] = diff['dst_mode']
1214                 os.unlink(dest)
1215                 editedFiles.add(dest)
1216             elif modifier == "R":
1217                 src, dest = diff['src'], diff['dst']
1218                 if self.p4HasMoveCommand:
1219                     p4_edit(src)        # src must be open before move
1220                     p4_move(src, dest)  # opens for (move/delete, move/add)
1221                 else:
1222                     p4_integrate(src, dest)
1223                     if diff['src_sha1'] != diff['dst_sha1']:
1224                         p4_edit(dest)
1225                     else:
1226                         pureRenameCopy.add(dest)
1227                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1228                     if not self.p4HasMoveCommand:
1229                         p4_edit(dest)   # with move: already open, writable
1230                     filesToChangeExecBit[dest] = diff['dst_mode']
1231                 if not self.p4HasMoveCommand:
1232                     os.unlink(dest)
1233                     filesToDelete.add(src)
1234                 editedFiles.add(dest)
1235             else:
1236                 die("unknown modifier %s for %s" % (modifier, path))
1237
1238         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1239         patchcmd = diffcmd + " | git apply "
1240         tryPatchCmd = patchcmd + "--check -"
1241         applyPatchCmd = patchcmd + "--check --apply -"
1242         patch_succeeded = True
1243
1244         if os.system(tryPatchCmd) != 0:
1245             fixed_rcs_keywords = False
1246             patch_succeeded = False
1247             print "Unfortunately applying the change failed!"
1248
1249             # Patch failed, maybe it's just RCS keyword woes. Look through
1250             # the patch to see if that's possible.
1251             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1252                 file = None
1253                 pattern = None
1254                 kwfiles = {}
1255                 for file in editedFiles | filesToDelete:
1256                     # did this file's delta contain RCS keywords?
1257                     pattern = p4_keywords_regexp_for_file(file)
1258
1259                     if pattern:
1260                         # this file is a possibility...look for RCS keywords.
1261                         regexp = re.compile(pattern, re.VERBOSE)
1262                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1263                             if regexp.search(line):
1264                                 if verbose:
1265                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1266                                 kwfiles[file] = pattern
1267                                 break
1268
1269                 for file in kwfiles:
1270                     if verbose:
1271                         print "zapping %s with %s" % (line,pattern)
1272                     self.patchRCSKeywords(file, kwfiles[file])
1273                     fixed_rcs_keywords = True
1274
1275             if fixed_rcs_keywords:
1276                 print "Retrying the patch with RCS keywords cleaned up"
1277                 if os.system(tryPatchCmd) == 0:
1278                     patch_succeeded = True
1279
1280         if not patch_succeeded:
1281             for f in editedFiles:
1282                 p4_revert(f)
1283             return False
1284
1285         #
1286         # Apply the patch for real, and do add/delete/+x handling.
1287         #
1288         system(applyPatchCmd)
1289
1290         for f in filesToAdd:
1291             p4_add(f)
1292         for f in filesToDelete:
1293             p4_revert(f)
1294             p4_delete(f)
1295
1296         # Set/clear executable bits
1297         for f in filesToChangeExecBit.keys():
1298             mode = filesToChangeExecBit[f]
1299             setP4ExecBit(f, mode)
1300
1301         #
1302         # Build p4 change description, starting with the contents
1303         # of the git commit message.
1304         #
1305         logMessage = extractLogMessageFromGitCommit(id)
1306         logMessage = logMessage.strip()
1307         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1308
1309         template = self.prepareSubmitTemplate()
1310         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1311
1312         if self.preserveUser:
1313            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1314
1315         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1316             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1317             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1318             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1319
1320         separatorLine = "######## everything below this line is just the diff #######\n"
1321
1322         # diff
1323         if os.environ.has_key("P4DIFF"):
1324             del(os.environ["P4DIFF"])
1325         diff = ""
1326         for editedFile in editedFiles:
1327             diff += p4_read_pipe(['diff', '-du',
1328                                   wildcard_encode(editedFile)])
1329
1330         # new file diff
1331         newdiff = ""
1332         for newFile in filesToAdd:
1333             newdiff += "==== new file ====\n"
1334             newdiff += "--- /dev/null\n"
1335             newdiff += "+++ %s\n" % newFile
1336             f = open(newFile, "r")
1337             for line in f.readlines():
1338                 newdiff += "+" + line
1339             f.close()
1340
1341         # change description file: submitTemplate, separatorLine, diff, newdiff
1342         (handle, fileName) = tempfile.mkstemp()
1343         tmpFile = os.fdopen(handle, "w+")
1344         if self.isWindows:
1345             submitTemplate = submitTemplate.replace("\n", "\r\n")
1346             separatorLine = separatorLine.replace("\n", "\r\n")
1347             newdiff = newdiff.replace("\n", "\r\n")
1348         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1349         tmpFile.close()
1350
1351         if self.prepare_p4_only:
1352             #
1353             # Leave the p4 tree prepared, and the submit template around
1354             # and let the user decide what to do next
1355             #
1356             print
1357             print "P4 workspace prepared for submission."
1358             print "To submit or revert, go to client workspace"
1359             print "  " + self.clientPath
1360             print
1361             print "To submit, use \"p4 submit\" to write a new description,"
1362             print "or \"p4 submit -i %s\" to use the one prepared by" \
1363                   " \"git p4\"." % fileName
1364             print "You can delete the file \"%s\" when finished." % fileName
1365
1366             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1367                 print "To preserve change ownership by user %s, you must\n" \
1368                       "do \"p4 change -f <change>\" after submitting and\n" \
1369                       "edit the User field."
1370             if pureRenameCopy:
1371                 print "After submitting, renamed files must be re-synced."
1372                 print "Invoke \"p4 sync -f\" on each of these files:"
1373                 for f in pureRenameCopy:
1374                     print "  " + f
1375
1376             print
1377             print "To revert the changes, use \"p4 revert ...\", and delete"
1378             print "the submit template file \"%s\"" % fileName
1379             if filesToAdd:
1380                 print "Since the commit adds new files, they must be deleted:"
1381                 for f in filesToAdd:
1382                     print "  " + f
1383             print
1384             return True
1385
1386         #
1387         # Let the user edit the change description, then submit it.
1388         #
1389         if self.edit_template(fileName):
1390             # read the edited message and submit
1391             ret = True
1392             tmpFile = open(fileName, "rb")
1393             message = tmpFile.read()
1394             tmpFile.close()
1395             submitTemplate = message[:message.index(separatorLine)]
1396             if self.isWindows:
1397                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1398             p4_write_pipe(['submit', '-i'], submitTemplate)
1399
1400             if self.preserveUser:
1401                 if p4User:
1402                     # Get last changelist number. Cannot easily get it from
1403                     # the submit command output as the output is
1404                     # unmarshalled.
1405                     changelist = self.lastP4Changelist()
1406                     self.modifyChangelistUser(changelist, p4User)
1407
1408             # The rename/copy happened by applying a patch that created a
1409             # new file.  This leaves it writable, which confuses p4.
1410             for f in pureRenameCopy:
1411                 p4_sync(f, "-f")
1412
1413         else:
1414             # skip this patch
1415             ret = False
1416             print "Submission cancelled, undoing p4 changes."
1417             for f in editedFiles:
1418                 p4_revert(f)
1419             for f in filesToAdd:
1420                 p4_revert(f)
1421                 os.remove(f)
1422             for f in filesToDelete:
1423                 p4_revert(f)
1424
1425         os.remove(fileName)
1426         return ret
1427
1428     # Export git tags as p4 labels. Create a p4 label and then tag
1429     # with that.
1430     def exportGitTags(self, gitTags):
1431         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1432         if len(validLabelRegexp) == 0:
1433             validLabelRegexp = defaultLabelRegexp
1434         m = re.compile(validLabelRegexp)
1435
1436         for name in gitTags:
1437
1438             if not m.match(name):
1439                 if verbose:
1440                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1441                 continue
1442
1443             # Get the p4 commit this corresponds to
1444             logMessage = extractLogMessageFromGitCommit(name)
1445             values = extractSettingsGitLog(logMessage)
1446
1447             if not values.has_key('change'):
1448                 # a tag pointing to something not sent to p4; ignore
1449                 if verbose:
1450                     print "git tag %s does not give a p4 commit" % name
1451                 continue
1452             else:
1453                 changelist = values['change']
1454
1455             # Get the tag details.
1456             inHeader = True
1457             isAnnotated = False
1458             body = []
1459             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1460                 l = l.strip()
1461                 if inHeader:
1462                     if re.match(r'tag\s+', l):
1463                         isAnnotated = True
1464                     elif re.match(r'\s*$', l):
1465                         inHeader = False
1466                         continue
1467                 else:
1468                     body.append(l)
1469
1470             if not isAnnotated:
1471                 body = ["lightweight tag imported by git p4\n"]
1472
1473             # Create the label - use the same view as the client spec we are using
1474             clientSpec = getClientSpec()
1475
1476             labelTemplate  = "Label: %s\n" % name
1477             labelTemplate += "Description:\n"
1478             for b in body:
1479                 labelTemplate += "\t" + b + "\n"
1480             labelTemplate += "View:\n"
1481             for mapping in clientSpec.mappings:
1482                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1483
1484             if self.dry_run:
1485                 print "Would create p4 label %s for tag" % name
1486             elif self.prepare_p4_only:
1487                 print "Not creating p4 label %s for tag due to option" \
1488                       " --prepare-p4-only" % name
1489             else:
1490                 p4_write_pipe(["label", "-i"], labelTemplate)
1491
1492                 # Use the label
1493                 p4_system(["tag", "-l", name] +
1494                           ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1495
1496                 if verbose:
1497                     print "created p4 label for tag %s" % name
1498
1499     def run(self, args):
1500         if len(args) == 0:
1501             self.master = currentGitBranch()
1502             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1503                 die("Detecting current git branch failed!")
1504         elif len(args) == 1:
1505             self.master = args[0]
1506             if not branchExists(self.master):
1507                 die("Branch %s does not exist" % self.master)
1508         else:
1509             return False
1510
1511         allowSubmit = gitConfig("git-p4.allowSubmit")
1512         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1513             die("%s is not in git-p4.allowSubmit" % self.master)
1514
1515         [upstream, settings] = findUpstreamBranchPoint()
1516         self.depotPath = settings['depot-paths'][0]
1517         if len(self.origin) == 0:
1518             self.origin = upstream
1519
1520         if self.preserveUser:
1521             if not self.canChangeChangelists():
1522                 die("Cannot preserve user names without p4 super-user or admin permissions")
1523
1524         # if not set from the command line, try the config file
1525         if self.conflict_behavior is None:
1526             val = gitConfig("git-p4.conflict")
1527             if val:
1528                 if val not in self.conflict_behavior_choices:
1529                     die("Invalid value '%s' for config git-p4.conflict" % val)
1530             else:
1531                 val = "ask"
1532             self.conflict_behavior = val
1533
1534         if self.verbose:
1535             print "Origin branch is " + self.origin
1536
1537         if len(self.depotPath) == 0:
1538             print "Internal error: cannot locate perforce depot path from existing branches"
1539             sys.exit(128)
1540
1541         self.useClientSpec = False
1542         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1543             self.useClientSpec = True
1544         if self.useClientSpec:
1545             self.clientSpecDirs = getClientSpec()
1546
1547         if self.useClientSpec:
1548             # all files are relative to the client spec
1549             self.clientPath = getClientRoot()
1550         else:
1551             self.clientPath = p4Where(self.depotPath)
1552
1553         if self.clientPath == "":
1554             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1555
1556         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1557         self.oldWorkingDirectory = os.getcwd()
1558
1559         # ensure the clientPath exists
1560         new_client_dir = False
1561         if not os.path.exists(self.clientPath):
1562             new_client_dir = True
1563             os.makedirs(self.clientPath)
1564
1565         chdir(self.clientPath)
1566         if self.dry_run:
1567             print "Would synchronize p4 checkout in %s" % self.clientPath
1568         else:
1569             print "Synchronizing p4 checkout..."
1570             if new_client_dir:
1571                 # old one was destroyed, and maybe nobody told p4
1572                 p4_sync("...", "-f")
1573             else:
1574                 p4_sync("...")
1575         self.check()
1576
1577         commits = []
1578         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1579             commits.append(line.strip())
1580         commits.reverse()
1581
1582         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1583             self.checkAuthorship = False
1584         else:
1585             self.checkAuthorship = True
1586
1587         if self.preserveUser:
1588             self.checkValidP4Users(commits)
1589
1590         #
1591         # Build up a set of options to be passed to diff when
1592         # submitting each commit to p4.
1593         #
1594         if self.detectRenames:
1595             # command-line -M arg
1596             self.diffOpts = "-M"
1597         else:
1598             # If not explicitly set check the config variable
1599             detectRenames = gitConfig("git-p4.detectRenames")
1600
1601             if detectRenames.lower() == "false" or detectRenames == "":
1602                 self.diffOpts = ""
1603             elif detectRenames.lower() == "true":
1604                 self.diffOpts = "-M"
1605             else:
1606                 self.diffOpts = "-M%s" % detectRenames
1607
1608         # no command-line arg for -C or --find-copies-harder, just
1609         # config variables
1610         detectCopies = gitConfig("git-p4.detectCopies")
1611         if detectCopies.lower() == "false" or detectCopies == "":
1612             pass
1613         elif detectCopies.lower() == "true":
1614             self.diffOpts += " -C"
1615         else:
1616             self.diffOpts += " -C%s" % detectCopies
1617
1618         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1619             self.diffOpts += " --find-copies-harder"
1620
1621         #
1622         # Apply the commits, one at a time.  On failure, ask if should
1623         # continue to try the rest of the patches, or quit.
1624         #
1625         if self.dry_run:
1626             print "Would apply"
1627         applied = []
1628         last = len(commits) - 1
1629         for i, commit in enumerate(commits):
1630             if self.dry_run:
1631                 print " ", read_pipe(["git", "show", "-s",
1632                                       "--format=format:%h %s", commit])
1633                 ok = True
1634             else:
1635                 ok = self.applyCommit(commit)
1636             if ok:
1637                 applied.append(commit)
1638             else:
1639                 if self.prepare_p4_only and i < last:
1640                     print "Processing only the first commit due to option" \
1641                           " --prepare-p4-only"
1642                     break
1643                 if i < last:
1644                     quit = False
1645                     while True:
1646                         # prompt for what to do, or use the option/variable
1647                         if self.conflict_behavior == "ask":
1648                             print "What do you want to do?"
1649                             response = raw_input("[s]kip this commit but apply"
1650                                                  " the rest, or [q]uit? ")
1651                             if not response:
1652                                 continue
1653                         elif self.conflict_behavior == "skip":
1654                             response = "s"
1655                         elif self.conflict_behavior == "quit":
1656                             response = "q"
1657                         else:
1658                             die("Unknown conflict_behavior '%s'" %
1659                                 self.conflict_behavior)
1660
1661                         if response[0] == "s":
1662                             print "Skipping this commit, but applying the rest"
1663                             break
1664                         if response[0] == "q":
1665                             print "Quitting"
1666                             quit = True
1667                             break
1668                     if quit:
1669                         break
1670
1671         chdir(self.oldWorkingDirectory)
1672
1673         if self.dry_run:
1674             pass
1675         elif self.prepare_p4_only:
1676             pass
1677         elif len(commits) == len(applied):
1678             print "All commits applied!"
1679
1680             sync = P4Sync()
1681             if self.branch:
1682                 sync.branch = self.branch
1683             sync.run([])
1684
1685             rebase = P4Rebase()
1686             rebase.rebase()
1687
1688         else:
1689             if len(applied) == 0:
1690                 print "No commits applied."
1691             else:
1692                 print "Applied only the commits marked with '*':"
1693                 for c in commits:
1694                     if c in applied:
1695                         star = "*"
1696                     else:
1697                         star = " "
1698                     print star, read_pipe(["git", "show", "-s",
1699                                            "--format=format:%h %s",  c])
1700                 print "You will have to do 'git p4 sync' and rebase."
1701
1702         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1703             self.exportLabels = True
1704
1705         if self.exportLabels:
1706             p4Labels = getP4Labels(self.depotPath)
1707             gitTags = getGitTags()
1708
1709             missingGitTags = gitTags - p4Labels
1710             self.exportGitTags(missingGitTags)
1711
1712         # exit with error unless everything applied perfecly
1713         if len(commits) != len(applied):
1714                 sys.exit(1)
1715
1716         return True
1717
1718 class View(object):
1719     """Represent a p4 view ("p4 help views"), and map files in a
1720        repo according to the view."""
1721
1722     class Path(object):
1723         """A depot or client path, possibly containing wildcards.
1724            The only one supported is ... at the end, currently.
1725            Initialize with the full path, with //depot or //client."""
1726
1727         def __init__(self, path, is_depot):
1728             self.path = path
1729             self.is_depot = is_depot
1730             self.find_wildcards()
1731             # remember the prefix bit, useful for relative mappings
1732             m = re.match("(//[^/]+/)", self.path)
1733             if not m:
1734                 die("Path %s does not start with //prefix/" % self.path)
1735             prefix = m.group(1)
1736             if not self.is_depot:
1737                 # strip //client/ on client paths
1738                 self.path = self.path[len(prefix):]
1739
1740         def find_wildcards(self):
1741             """Make sure wildcards are valid, and set up internal
1742                variables."""
1743
1744             self.ends_triple_dot = False
1745             # There are three wildcards allowed in p4 views
1746             # (see "p4 help views").  This code knows how to
1747             # handle "..." (only at the end), but cannot deal with
1748             # "%%n" or "*".  Only check the depot_side, as p4 should
1749             # validate that the client_side matches too.
1750             if re.search(r'%%[1-9]', self.path):
1751                 die("Can't handle %%n wildcards in view: %s" % self.path)
1752             if self.path.find("*") >= 0:
1753                 die("Can't handle * wildcards in view: %s" % self.path)
1754             triple_dot_index = self.path.find("...")
1755             if triple_dot_index >= 0:
1756                 if triple_dot_index != len(self.path) - 3:
1757                     die("Can handle only single ... wildcard, at end: %s" %
1758                         self.path)
1759                 self.ends_triple_dot = True
1760
1761         def ensure_compatible(self, other_path):
1762             """Make sure the wildcards agree."""
1763             if self.ends_triple_dot != other_path.ends_triple_dot:
1764                  die("Both paths must end with ... if either does;\n" +
1765                      "paths: %s %s" % (self.path, other_path.path))
1766
1767         def match_wildcards(self, test_path):
1768             """See if this test_path matches us, and fill in the value
1769                of the wildcards if so.  Returns a tuple of
1770                (True|False, wildcards[]).  For now, only the ... at end
1771                is supported, so at most one wildcard."""
1772             if self.ends_triple_dot:
1773                 dotless = self.path[:-3]
1774                 if test_path.startswith(dotless):
1775                     wildcard = test_path[len(dotless):]
1776                     return (True, [ wildcard ])
1777             else:
1778                 if test_path == self.path:
1779                     return (True, [])
1780             return (False, [])
1781
1782         def match(self, test_path):
1783             """Just return if it matches; don't bother with the wildcards."""
1784             b, _ = self.match_wildcards(test_path)
1785             return b
1786
1787         def fill_in_wildcards(self, wildcards):
1788             """Return the relative path, with the wildcards filled in
1789                if there are any."""
1790             if self.ends_triple_dot:
1791                 return self.path[:-3] + wildcards[0]
1792             else:
1793                 return self.path
1794
1795     class Mapping(object):
1796         def __init__(self, depot_side, client_side, overlay, exclude):
1797             # depot_side is without the trailing /... if it had one
1798             self.depot_side = View.Path(depot_side, is_depot=True)
1799             self.client_side = View.Path(client_side, is_depot=False)
1800             self.overlay = overlay  # started with "+"
1801             self.exclude = exclude  # started with "-"
1802             assert not (self.overlay and self.exclude)
1803             self.depot_side.ensure_compatible(self.client_side)
1804
1805         def __str__(self):
1806             c = " "
1807             if self.overlay:
1808                 c = "+"
1809             if self.exclude:
1810                 c = "-"
1811             return "View.Mapping: %s%s -> %s" % \
1812                    (c, self.depot_side.path, self.client_side.path)
1813
1814         def map_depot_to_client(self, depot_path):
1815             """Calculate the client path if using this mapping on the
1816                given depot path; does not consider the effect of other
1817                mappings in a view.  Even excluded mappings are returned."""
1818             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1819             if not matches:
1820                 return ""
1821             client_path = self.client_side.fill_in_wildcards(wildcards)
1822             return client_path
1823
1824     #
1825     # View methods
1826     #
1827     def __init__(self):
1828         self.mappings = []
1829
1830     def append(self, view_line):
1831         """Parse a view line, splitting it into depot and client
1832            sides.  Append to self.mappings, preserving order."""
1833
1834         # Split the view line into exactly two words.  P4 enforces
1835         # structure on these lines that simplifies this quite a bit.
1836         #
1837         # Either or both words may be double-quoted.
1838         # Single quotes do not matter.
1839         # Double-quote marks cannot occur inside the words.
1840         # A + or - prefix is also inside the quotes.
1841         # There are no quotes unless they contain a space.
1842         # The line is already white-space stripped.
1843         # The two words are separated by a single space.
1844         #
1845         if view_line[0] == '"':
1846             # First word is double quoted.  Find its end.
1847             close_quote_index = view_line.find('"', 1)
1848             if close_quote_index <= 0:
1849                 die("No first-word closing quote found: %s" % view_line)
1850             depot_side = view_line[1:close_quote_index]
1851             # skip closing quote and space
1852             rhs_index = close_quote_index + 1 + 1
1853         else:
1854             space_index = view_line.find(" ")
1855             if space_index <= 0:
1856                 die("No word-splitting space found: %s" % view_line)
1857             depot_side = view_line[0:space_index]
1858             rhs_index = space_index + 1
1859
1860         if view_line[rhs_index] == '"':
1861             # Second word is double quoted.  Make sure there is a
1862             # double quote at the end too.
1863             if not view_line.endswith('"'):
1864                 die("View line with rhs quote should end with one: %s" %
1865                     view_line)
1866             # skip the quotes
1867             client_side = view_line[rhs_index+1:-1]
1868         else:
1869             client_side = view_line[rhs_index:]
1870
1871         # prefix + means overlay on previous mapping
1872         overlay = False
1873         if depot_side.startswith("+"):
1874             overlay = True
1875             depot_side = depot_side[1:]
1876
1877         # prefix - means exclude this path
1878         exclude = False
1879         if depot_side.startswith("-"):
1880             exclude = True
1881             depot_side = depot_side[1:]
1882
1883         m = View.Mapping(depot_side, client_side, overlay, exclude)
1884         self.mappings.append(m)
1885
1886     def map_in_client(self, depot_path):
1887         """Return the relative location in the client where this
1888            depot file should live.  Returns "" if the file should
1889            not be mapped in the client."""
1890
1891         paths_filled = []
1892         client_path = ""
1893
1894         # look at later entries first
1895         for m in self.mappings[::-1]:
1896
1897             # see where will this path end up in the client
1898             p = m.map_depot_to_client(depot_path)
1899
1900             if p == "":
1901                 # Depot path does not belong in client.  Must remember
1902                 # this, as previous items should not cause files to
1903                 # exist in this path either.  Remember that the list is
1904                 # being walked from the end, which has higher precedence.
1905                 # Overlap mappings do not exclude previous mappings.
1906                 if not m.overlay:
1907                     paths_filled.append(m.client_side)
1908
1909             else:
1910                 # This mapping matched; no need to search any further.
1911                 # But, the mapping could be rejected if the client path
1912                 # has already been claimed by an earlier mapping (i.e.
1913                 # one later in the list, which we are walking backwards).
1914                 already_mapped_in_client = False
1915                 for f in paths_filled:
1916                     # this is View.Path.match
1917                     if f.match(p):
1918                         already_mapped_in_client = True
1919                         break
1920                 if not already_mapped_in_client:
1921                     # Include this file, unless it is from a line that
1922                     # explicitly said to exclude it.
1923                     if not m.exclude:
1924                         client_path = p
1925
1926                 # a match, even if rejected, always stops the search
1927                 break
1928
1929         return client_path
1930
1931 class P4Sync(Command, P4UserMap):
1932     delete_actions = ( "delete", "move/delete", "purge" )
1933
1934     def __init__(self):
1935         Command.__init__(self)
1936         P4UserMap.__init__(self)
1937         self.options = [
1938                 optparse.make_option("--branch", dest="branch"),
1939                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1940                 optparse.make_option("--changesfile", dest="changesFile"),
1941                 optparse.make_option("--silent", dest="silent", action="store_true"),
1942                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1943                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1944                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1945                                      help="Import into refs/heads/ , not refs/remotes"),
1946                 optparse.make_option("--max-changes", dest="maxChanges"),
1947                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1948                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1949                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1950                                      help="Only sync files that are included in the Perforce Client Spec")
1951         ]
1952         self.description = """Imports from Perforce into a git repository.\n
1953     example:
1954     //depot/my/project/ -- to import the current head
1955     //depot/my/project/@all -- to import everything
1956     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1957
1958     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1959
1960         self.usage += " //depot/path[@revRange]"
1961         self.silent = False
1962         self.createdBranches = set()
1963         self.committedChanges = set()
1964         self.branch = ""
1965         self.detectBranches = False
1966         self.detectLabels = False
1967         self.importLabels = False
1968         self.changesFile = ""
1969         self.syncWithOrigin = True
1970         self.importIntoRemotes = True
1971         self.maxChanges = ""
1972         self.isWindows = (platform.system() == "Windows")
1973         self.keepRepoPath = False
1974         self.depotPaths = None
1975         self.p4BranchesInGit = []
1976         self.cloneExclude = []
1977         self.useClientSpec = False
1978         self.useClientSpec_from_options = False
1979         self.clientSpecDirs = None
1980         self.tempBranches = []
1981         self.tempBranchLocation = "git-p4-tmp"
1982
1983         if gitConfig("git-p4.syncFromOrigin") == "false":
1984             self.syncWithOrigin = False
1985
1986     # Force a checkpoint in fast-import and wait for it to finish
1987     def checkpoint(self):
1988         self.gitStream.write("checkpoint\n\n")
1989         self.gitStream.write("progress checkpoint\n\n")
1990         out = self.gitOutput.readline()
1991         if self.verbose:
1992             print "checkpoint finished: " + out
1993
1994     def extractFilesFromCommit(self, commit):
1995         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1996                              for path in self.cloneExclude]
1997         files = []
1998         fnum = 0
1999         while commit.has_key("depotFile%s" % fnum):
2000             path =  commit["depotFile%s" % fnum]
2001
2002             if [p for p in self.cloneExclude
2003                 if p4PathStartsWith(path, p)]:
2004                 found = False
2005             else:
2006                 found = [p for p in self.depotPaths
2007                          if p4PathStartsWith(path, p)]
2008             if not found:
2009                 fnum = fnum + 1
2010                 continue
2011
2012             file = {}
2013             file["path"] = path
2014             file["rev"] = commit["rev%s" % fnum]
2015             file["action"] = commit["action%s" % fnum]
2016             file["type"] = commit["type%s" % fnum]
2017             files.append(file)
2018             fnum = fnum + 1
2019         return files
2020
2021     def stripRepoPath(self, path, prefixes):
2022         """When streaming files, this is called to map a p4 depot path
2023            to where it should go in git.  The prefixes are either
2024            self.depotPaths, or self.branchPrefixes in the case of
2025            branch detection."""
2026
2027         if self.useClientSpec:
2028             # branch detection moves files up a level (the branch name)
2029             # from what client spec interpretation gives
2030             path = self.clientSpecDirs.map_in_client(path)
2031             if self.detectBranches:
2032                 for b in self.knownBranches:
2033                     if path.startswith(b + "/"):
2034                         path = path[len(b)+1:]
2035
2036         elif self.keepRepoPath:
2037             # Preserve everything in relative path name except leading
2038             # //depot/; just look at first prefix as they all should
2039             # be in the same depot.
2040             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2041             if p4PathStartsWith(path, depot):
2042                 path = path[len(depot):]
2043
2044         else:
2045             for p in prefixes:
2046                 if p4PathStartsWith(path, p):
2047                     path = path[len(p):]
2048                     break
2049
2050         path = wildcard_decode(path)
2051         return path
2052
2053     def splitFilesIntoBranches(self, commit):
2054         """Look at each depotFile in the commit to figure out to what
2055            branch it belongs."""
2056
2057         branches = {}
2058         fnum = 0
2059         while commit.has_key("depotFile%s" % fnum):
2060             path =  commit["depotFile%s" % fnum]
2061             found = [p for p in self.depotPaths
2062                      if p4PathStartsWith(path, p)]
2063             if not found:
2064                 fnum = fnum + 1
2065                 continue
2066
2067             file = {}
2068             file["path"] = path
2069             file["rev"] = commit["rev%s" % fnum]
2070             file["action"] = commit["action%s" % fnum]
2071             file["type"] = commit["type%s" % fnum]
2072             fnum = fnum + 1
2073
2074             # start with the full relative path where this file would
2075             # go in a p4 client
2076             if self.useClientSpec:
2077                 relPath = self.clientSpecDirs.map_in_client(path)
2078             else:
2079                 relPath = self.stripRepoPath(path, self.depotPaths)
2080
2081             for branch in self.knownBranches.keys():
2082                 # add a trailing slash so that a commit into qt/4.2foo
2083                 # doesn't end up in qt/4.2, e.g.
2084                 if relPath.startswith(branch + "/"):
2085                     if branch not in branches:
2086                         branches[branch] = []
2087                     branches[branch].append(file)
2088                     break
2089
2090         return branches
2091
2092     # output one file from the P4 stream
2093     # - helper for streamP4Files
2094
2095     def streamOneP4File(self, file, contents):
2096         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2097         if verbose:
2098             sys.stderr.write("%s\n" % relPath)
2099
2100         (type_base, type_mods) = split_p4_type(file["type"])
2101
2102         git_mode = "100644"
2103         if "x" in type_mods:
2104             git_mode = "100755"
2105         if type_base == "symlink":
2106             git_mode = "120000"
2107             # p4 print on a symlink contains "target\n"; remove the newline
2108             data = ''.join(contents)
2109             contents = [data[:-1]]
2110
2111         if type_base == "utf16":
2112             # p4 delivers different text in the python output to -G
2113             # than it does when using "print -o", or normal p4 client
2114             # operations.  utf16 is converted to ascii or utf8, perhaps.
2115             # But ascii text saved as -t utf16 is completely mangled.
2116             # Invoke print -o to get the real contents.
2117             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2118             contents = [ text ]
2119
2120         if type_base == "apple":
2121             # Apple filetype files will be streamed as a concatenation of
2122             # its appledouble header and the contents.  This is useless
2123             # on both macs and non-macs.  If using "print -q -o xx", it
2124             # will create "xx" with the data, and "%xx" with the header.
2125             # This is also not very useful.
2126             #
2127             # Ideally, someday, this script can learn how to generate
2128             # appledouble files directly and import those to git, but
2129             # non-mac machines can never find a use for apple filetype.
2130             print "\nIgnoring apple filetype file %s" % file['depotFile']
2131             return
2132
2133         # Perhaps windows wants unicode, utf16 newlines translated too;
2134         # but this is not doing it.
2135         if self.isWindows and type_base == "text":
2136             mangled = []
2137             for data in contents:
2138                 data = data.replace("\r\n", "\n")
2139                 mangled.append(data)
2140             contents = mangled
2141
2142         # Note that we do not try to de-mangle keywords on utf16 files,
2143         # even though in theory somebody may want that.
2144         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2145         if pattern:
2146             regexp = re.compile(pattern, re.VERBOSE)
2147             text = ''.join(contents)
2148             text = regexp.sub(r'$\1$', text)
2149             contents = [ text ]
2150
2151         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2152
2153         # total length...
2154         length = 0
2155         for d in contents:
2156             length = length + len(d)
2157
2158         self.gitStream.write("data %d\n" % length)
2159         for d in contents:
2160             self.gitStream.write(d)
2161         self.gitStream.write("\n")
2162
2163     def streamOneP4Deletion(self, file):
2164         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2165         if verbose:
2166             sys.stderr.write("delete %s\n" % relPath)
2167         self.gitStream.write("D %s\n" % relPath)
2168
2169     # handle another chunk of streaming data
2170     def streamP4FilesCb(self, marshalled):
2171
2172         # catch p4 errors and complain
2173         err = None
2174         if "code" in marshalled:
2175             if marshalled["code"] == "error":
2176                 if "data" in marshalled:
2177                     err = marshalled["data"].rstrip()
2178         if err:
2179             f = None
2180             if self.stream_have_file_info:
2181                 if "depotFile" in self.stream_file:
2182                     f = self.stream_file["depotFile"]
2183             # force a failure in fast-import, else an empty
2184             # commit will be made
2185             self.gitStream.write("\n")
2186             self.gitStream.write("die-now\n")
2187             self.gitStream.close()
2188             # ignore errors, but make sure it exits first
2189             self.importProcess.wait()
2190             if f:
2191                 die("Error from p4 print for %s: %s" % (f, err))
2192             else:
2193                 die("Error from p4 print: %s" % err)
2194
2195         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2196             # start of a new file - output the old one first
2197             self.streamOneP4File(self.stream_file, self.stream_contents)
2198             self.stream_file = {}
2199             self.stream_contents = []
2200             self.stream_have_file_info = False
2201
2202         # pick up the new file information... for the
2203         # 'data' field we need to append to our array
2204         for k in marshalled.keys():
2205             if k == 'data':
2206                 self.stream_contents.append(marshalled['data'])
2207             else:
2208                 self.stream_file[k] = marshalled[k]
2209
2210         self.stream_have_file_info = True
2211
2212     # Stream directly from "p4 files" into "git fast-import"
2213     def streamP4Files(self, files):
2214         filesForCommit = []
2215         filesToRead = []
2216         filesToDelete = []
2217
2218         for f in files:
2219             # if using a client spec, only add the files that have
2220             # a path in the client
2221             if self.clientSpecDirs:
2222                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2223                     continue
2224
2225             filesForCommit.append(f)
2226             if f['action'] in self.delete_actions:
2227                 filesToDelete.append(f)
2228             else:
2229                 filesToRead.append(f)
2230
2231         # deleted files...
2232         for f in filesToDelete:
2233             self.streamOneP4Deletion(f)
2234
2235         if len(filesToRead) > 0:
2236             self.stream_file = {}
2237             self.stream_contents = []
2238             self.stream_have_file_info = False
2239
2240             # curry self argument
2241             def streamP4FilesCbSelf(entry):
2242                 self.streamP4FilesCb(entry)
2243
2244             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2245
2246             p4CmdList(["-x", "-", "print"],
2247                       stdin=fileArgs,
2248                       cb=streamP4FilesCbSelf)
2249
2250             # do the last chunk
2251             if self.stream_file.has_key('depotFile'):
2252                 self.streamOneP4File(self.stream_file, self.stream_contents)
2253
2254     def make_email(self, userid):
2255         if userid in self.users:
2256             return self.users[userid]
2257         else:
2258             return "%s <a@b>" % userid
2259
2260     # Stream a p4 tag
2261     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2262         if verbose:
2263             print "writing tag %s for commit %s" % (labelName, commit)
2264         gitStream.write("tag %s\n" % labelName)
2265         gitStream.write("from %s\n" % commit)
2266
2267         if labelDetails.has_key('Owner'):
2268             owner = labelDetails["Owner"]
2269         else:
2270             owner = None
2271
2272         # Try to use the owner of the p4 label, or failing that,
2273         # the current p4 user id.
2274         if owner:
2275             email = self.make_email(owner)
2276         else:
2277             email = self.make_email(self.p4UserId())
2278         tagger = "%s %s %s" % (email, epoch, self.tz)
2279
2280         gitStream.write("tagger %s\n" % tagger)
2281
2282         print "labelDetails=",labelDetails
2283         if labelDetails.has_key('Description'):
2284             description = labelDetails['Description']
2285         else:
2286             description = 'Label from git p4'
2287
2288         gitStream.write("data %d\n" % len(description))
2289         gitStream.write(description)
2290         gitStream.write("\n")
2291
2292     def commit(self, details, files, branch, parent = ""):
2293         epoch = details["time"]
2294         author = details["user"]
2295
2296         if self.verbose:
2297             print "commit into %s" % branch
2298
2299         # start with reading files; if that fails, we should not
2300         # create a commit.
2301         new_files = []
2302         for f in files:
2303             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2304                 new_files.append (f)
2305             else:
2306                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2307
2308         self.gitStream.write("commit %s\n" % branch)
2309 #        gitStream.write("mark :%s\n" % details["change"])
2310         self.committedChanges.add(int(details["change"]))
2311         committer = ""
2312         if author not in self.users:
2313             self.getUserMapFromPerforceServer()
2314         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2315
2316         self.gitStream.write("committer %s\n" % committer)
2317
2318         self.gitStream.write("data <<EOT\n")
2319         self.gitStream.write(details["desc"])
2320         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2321                              (','.join(self.branchPrefixes), details["change"]))
2322         if len(details['options']) > 0:
2323             self.gitStream.write(": options = %s" % details['options'])
2324         self.gitStream.write("]\nEOT\n\n")
2325
2326         if len(parent) > 0:
2327             if self.verbose:
2328                 print "parent %s" % parent
2329             self.gitStream.write("from %s\n" % parent)
2330
2331         self.streamP4Files(new_files)
2332         self.gitStream.write("\n")
2333
2334         change = int(details["change"])
2335
2336         if self.labels.has_key(change):
2337             label = self.labels[change]
2338             labelDetails = label[0]
2339             labelRevisions = label[1]
2340             if self.verbose:
2341                 print "Change %s is labelled %s" % (change, labelDetails)
2342
2343             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2344                                                 for p in self.branchPrefixes])
2345
2346             if len(files) == len(labelRevisions):
2347
2348                 cleanedFiles = {}
2349                 for info in files:
2350                     if info["action"] in self.delete_actions:
2351                         continue
2352                     cleanedFiles[info["depotFile"]] = info["rev"]
2353
2354                 if cleanedFiles == labelRevisions:
2355                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2356
2357                 else:
2358                     if not self.silent:
2359                         print ("Tag %s does not match with change %s: files do not match."
2360                                % (labelDetails["label"], change))
2361
2362             else:
2363                 if not self.silent:
2364                     print ("Tag %s does not match with change %s: file count is different."
2365                            % (labelDetails["label"], change))
2366
2367     # Build a dictionary of changelists and labels, for "detect-labels" option.
2368     def getLabels(self):
2369         self.labels = {}
2370
2371         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2372         if len(l) > 0 and not self.silent:
2373             print "Finding files belonging to labels in %s" % `self.depotPaths`
2374
2375         for output in l:
2376             label = output["label"]
2377             revisions = {}
2378             newestChange = 0
2379             if self.verbose:
2380                 print "Querying files for label %s" % label
2381             for file in p4CmdList(["files"] +
2382                                       ["%s...@%s" % (p, label)
2383                                           for p in self.depotPaths]):
2384                 revisions[file["depotFile"]] = file["rev"]
2385                 change = int(file["change"])
2386                 if change > newestChange:
2387                     newestChange = change
2388
2389             self.labels[newestChange] = [output, revisions]
2390
2391         if self.verbose:
2392             print "Label changes: %s" % self.labels.keys()
2393
2394     # Import p4 labels as git tags. A direct mapping does not
2395     # exist, so assume that if all the files are at the same revision
2396     # then we can use that, or it's something more complicated we should
2397     # just ignore.
2398     def importP4Labels(self, stream, p4Labels):
2399         if verbose:
2400             print "import p4 labels: " + ' '.join(p4Labels)
2401
2402         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2403         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2404         if len(validLabelRegexp) == 0:
2405             validLabelRegexp = defaultLabelRegexp
2406         m = re.compile(validLabelRegexp)
2407
2408         for name in p4Labels:
2409             commitFound = False
2410
2411             if not m.match(name):
2412                 if verbose:
2413                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2414                 continue
2415
2416             if name in ignoredP4Labels:
2417                 continue
2418
2419             labelDetails = p4CmdList(['label', "-o", name])[0]
2420
2421             # get the most recent changelist for each file in this label
2422             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2423                                 for p in self.depotPaths])
2424
2425             if change.has_key('change'):
2426                 # find the corresponding git commit; take the oldest commit
2427                 changelist = int(change['change'])
2428                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2429                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2430                 if len(gitCommit) == 0:
2431                     print "could not find git commit for changelist %d" % changelist
2432                 else:
2433                     gitCommit = gitCommit.strip()
2434                     commitFound = True
2435                     # Convert from p4 time format
2436                     try:
2437                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2438                     except ValueError:
2439                         print "Could not convert label time %s" % labelDetails['Update']
2440                         tmwhen = 1
2441
2442                     when = int(time.mktime(tmwhen))
2443                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2444                     if verbose:
2445                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2446             else:
2447                 if verbose:
2448                     print "Label %s has no changelists - possibly deleted?" % name
2449
2450             if not commitFound:
2451                 # We can't import this label; don't try again as it will get very
2452                 # expensive repeatedly fetching all the files for labels that will
2453                 # never be imported. If the label is moved in the future, the
2454                 # ignore will need to be removed manually.
2455                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2456
2457     def guessProjectName(self):
2458         for p in self.depotPaths:
2459             if p.endswith("/"):
2460                 p = p[:-1]
2461             p = p[p.strip().rfind("/") + 1:]
2462             if not p.endswith("/"):
2463                p += "/"
2464             return p
2465
2466     def getBranchMapping(self):
2467         lostAndFoundBranches = set()
2468
2469         user = gitConfig("git-p4.branchUser")
2470         if len(user) > 0:
2471             command = "branches -u %s" % user
2472         else:
2473             command = "branches"
2474
2475         for info in p4CmdList(command):
2476             details = p4Cmd(["branch", "-o", info["branch"]])
2477             viewIdx = 0
2478             while details.has_key("View%s" % viewIdx):
2479                 paths = details["View%s" % viewIdx].split(" ")
2480                 viewIdx = viewIdx + 1
2481                 # require standard //depot/foo/... //depot/bar/... mapping
2482                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2483                     continue
2484                 source = paths[0]
2485                 destination = paths[1]
2486                 ## HACK
2487                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2488                     source = source[len(self.depotPaths[0]):-4]
2489                     destination = destination[len(self.depotPaths[0]):-4]
2490
2491                     if destination in self.knownBranches:
2492                         if not self.silent:
2493                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2494                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2495                         continue
2496
2497                     self.knownBranches[destination] = source
2498
2499                     lostAndFoundBranches.discard(destination)
2500
2501                     if source not in self.knownBranches:
2502                         lostAndFoundBranches.add(source)
2503
2504         # Perforce does not strictly require branches to be defined, so we also
2505         # check git config for a branch list.
2506         #
2507         # Example of branch definition in git config file:
2508         # [git-p4]
2509         #   branchList=main:branchA
2510         #   branchList=main:branchB
2511         #   branchList=branchA:branchC
2512         configBranches = gitConfigList("git-p4.branchList")
2513         for branch in configBranches:
2514             if branch:
2515                 (source, destination) = branch.split(":")
2516                 self.knownBranches[destination] = source
2517
2518                 lostAndFoundBranches.discard(destination)
2519
2520                 if source not in self.knownBranches:
2521                     lostAndFoundBranches.add(source)
2522
2523
2524         for branch in lostAndFoundBranches:
2525             self.knownBranches[branch] = branch
2526
2527     def getBranchMappingFromGitBranches(self):
2528         branches = p4BranchesInGit(self.importIntoRemotes)
2529         for branch in branches.keys():
2530             if branch == "master":
2531                 branch = "main"
2532             else:
2533                 branch = branch[len(self.projectName):]
2534             self.knownBranches[branch] = branch
2535
2536     def updateOptionDict(self, d):
2537         option_keys = {}
2538         if self.keepRepoPath:
2539             option_keys['keepRepoPath'] = 1
2540
2541         d["options"] = ' '.join(sorted(option_keys.keys()))
2542
2543     def readOptions(self, d):
2544         self.keepRepoPath = (d.has_key('options')
2545                              and ('keepRepoPath' in d['options']))
2546
2547     def gitRefForBranch(self, branch):
2548         if branch == "main":
2549             return self.refPrefix + "master"
2550
2551         if len(branch) <= 0:
2552             return branch
2553
2554         return self.refPrefix + self.projectName + branch
2555
2556     def gitCommitByP4Change(self, ref, change):
2557         if self.verbose:
2558             print "looking in ref " + ref + " for change %s using bisect..." % change
2559
2560         earliestCommit = ""
2561         latestCommit = parseRevision(ref)
2562
2563         while True:
2564             if self.verbose:
2565                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2566             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2567             if len(next) == 0:
2568                 if self.verbose:
2569                     print "argh"
2570                 return ""
2571             log = extractLogMessageFromGitCommit(next)
2572             settings = extractSettingsGitLog(log)
2573             currentChange = int(settings['change'])
2574             if self.verbose:
2575                 print "current change %s" % currentChange
2576
2577             if currentChange == change:
2578                 if self.verbose:
2579                     print "found %s" % next
2580                 return next
2581
2582             if currentChange < change:
2583                 earliestCommit = "^%s" % next
2584             else:
2585                 latestCommit = "%s" % next
2586
2587         return ""
2588
2589     def importNewBranch(self, branch, maxChange):
2590         # make fast-import flush all changes to disk and update the refs using the checkpoint
2591         # command so that we can try to find the branch parent in the git history
2592         self.gitStream.write("checkpoint\n\n");
2593         self.gitStream.flush();
2594         branchPrefix = self.depotPaths[0] + branch + "/"
2595         range = "@1,%s" % maxChange
2596         #print "prefix" + branchPrefix
2597         changes = p4ChangesForPaths([branchPrefix], range)
2598         if len(changes) <= 0:
2599             return False
2600         firstChange = changes[0]
2601         #print "first change in branch: %s" % firstChange
2602         sourceBranch = self.knownBranches[branch]
2603         sourceDepotPath = self.depotPaths[0] + sourceBranch
2604         sourceRef = self.gitRefForBranch(sourceBranch)
2605         #print "source " + sourceBranch
2606
2607         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2608         #print "branch parent: %s" % branchParentChange
2609         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2610         if len(gitParent) > 0:
2611             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2612             #print "parent git commit: %s" % gitParent
2613
2614         self.importChanges(changes)
2615         return True
2616
2617     def searchParent(self, parent, branch, target):
2618         parentFound = False
2619         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2620             blob = blob.strip()
2621             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2622                 parentFound = True
2623                 if self.verbose:
2624                     print "Found parent of %s in commit %s" % (branch, blob)
2625                 break
2626         if parentFound:
2627             return blob
2628         else:
2629             return None
2630
2631     def importChanges(self, changes):
2632         cnt = 1
2633         for change in changes:
2634             description = p4_describe(change)
2635             self.updateOptionDict(description)
2636
2637             if not self.silent:
2638                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2639                 sys.stdout.flush()
2640             cnt = cnt + 1
2641
2642             try:
2643                 if self.detectBranches:
2644                     branches = self.splitFilesIntoBranches(description)
2645                     for branch in branches.keys():
2646                         ## HACK  --hwn
2647                         branchPrefix = self.depotPaths[0] + branch + "/"
2648                         self.branchPrefixes = [ branchPrefix ]
2649
2650                         parent = ""
2651
2652                         filesForCommit = branches[branch]
2653
2654                         if self.verbose:
2655                             print "branch is %s" % branch
2656
2657                         self.updatedBranches.add(branch)
2658
2659                         if branch not in self.createdBranches:
2660                             self.createdBranches.add(branch)
2661                             parent = self.knownBranches[branch]
2662                             if parent == branch:
2663                                 parent = ""
2664                             else:
2665                                 fullBranch = self.projectName + branch
2666                                 if fullBranch not in self.p4BranchesInGit:
2667                                     if not self.silent:
2668                                         print("\n    Importing new branch %s" % fullBranch);
2669                                     if self.importNewBranch(branch, change - 1):
2670                                         parent = ""
2671                                         self.p4BranchesInGit.append(fullBranch)
2672                                     if not self.silent:
2673                                         print("\n    Resuming with change %s" % change);
2674
2675                                 if self.verbose:
2676                                     print "parent determined through known branches: %s" % parent
2677
2678                         branch = self.gitRefForBranch(branch)
2679                         parent = self.gitRefForBranch(parent)
2680
2681                         if self.verbose:
2682                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2683
2684                         if len(parent) == 0 and branch in self.initialParents:
2685                             parent = self.initialParents[branch]
2686                             del self.initialParents[branch]
2687
2688                         blob = None
2689                         if len(parent) > 0:
2690                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2691                             if self.verbose:
2692                                 print "Creating temporary branch: " + tempBranch
2693                             self.commit(description, filesForCommit, tempBranch)
2694                             self.tempBranches.append(tempBranch)
2695                             self.checkpoint()
2696                             blob = self.searchParent(parent, branch, tempBranch)
2697                         if blob:
2698                             self.commit(description, filesForCommit, branch, blob)
2699                         else:
2700                             if self.verbose:
2701                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2702                             self.commit(description, filesForCommit, branch, parent)
2703                 else:
2704                     files = self.extractFilesFromCommit(description)
2705                     self.commit(description, files, self.branch,
2706                                 self.initialParent)
2707                     # only needed once, to connect to the previous commit
2708                     self.initialParent = ""
2709             except IOError:
2710                 print self.gitError.read()
2711                 sys.exit(1)
2712
2713     def importHeadRevision(self, revision):
2714         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2715
2716         details = {}
2717         details["user"] = "git perforce import user"
2718         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2719                            % (' '.join(self.depotPaths), revision))
2720         details["change"] = revision
2721         newestRevision = 0
2722
2723         fileCnt = 0
2724         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2725
2726         for info in p4CmdList(["files"] + fileArgs):
2727
2728             if 'code' in info and info['code'] == 'error':
2729                 sys.stderr.write("p4 returned an error: %s\n"
2730                                  % info['data'])
2731                 if info['data'].find("must refer to client") >= 0:
2732                     sys.stderr.write("This particular p4 error is misleading.\n")
2733                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2734                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2735                 sys.exit(1)
2736             if 'p4ExitCode' in info:
2737                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2738                 sys.exit(1)
2739
2740
2741             change = int(info["change"])
2742             if change > newestRevision:
2743                 newestRevision = change
2744
2745             if info["action"] in self.delete_actions:
2746                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2747                 #fileCnt = fileCnt + 1
2748                 continue
2749
2750             for prop in ["depotFile", "rev", "action", "type" ]:
2751                 details["%s%s" % (prop, fileCnt)] = info[prop]
2752
2753             fileCnt = fileCnt + 1
2754
2755         details["change"] = newestRevision
2756
2757         # Use time from top-most change so that all git p4 clones of
2758         # the same p4 repo have the same commit SHA1s.
2759         res = p4_describe(newestRevision)
2760         details["time"] = res["time"]
2761
2762         self.updateOptionDict(details)
2763         try:
2764             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2765         except IOError:
2766             print "IO error with git fast-import. Is your git version recent enough?"
2767             print self.gitError.read()
2768
2769
2770     def run(self, args):
2771         self.depotPaths = []
2772         self.changeRange = ""
2773         self.previousDepotPaths = []
2774         self.hasOrigin = False
2775
2776         # map from branch depot path to parent branch
2777         self.knownBranches = {}
2778         self.initialParents = {}
2779
2780         if self.importIntoRemotes:
2781             self.refPrefix = "refs/remotes/p4/"
2782         else:
2783             self.refPrefix = "refs/heads/p4/"
2784
2785         if self.syncWithOrigin:
2786             self.hasOrigin = originP4BranchesExist()
2787             if self.hasOrigin:
2788                 if not self.silent:
2789                     print 'Syncing with origin first, using "git fetch origin"'
2790                 system("git fetch origin")
2791
2792         branch_arg_given = bool(self.branch)
2793         if len(self.branch) == 0:
2794             self.branch = self.refPrefix + "master"
2795             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2796                 system("git update-ref %s refs/heads/p4" % self.branch)
2797                 system("git branch -D p4")
2798
2799         # accept either the command-line option, or the configuration variable
2800         if self.useClientSpec:
2801             # will use this after clone to set the variable
2802             self.useClientSpec_from_options = True
2803         else:
2804             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2805                 self.useClientSpec = True
2806         if self.useClientSpec:
2807             self.clientSpecDirs = getClientSpec()
2808
2809         # TODO: should always look at previous commits,
2810         # merge with previous imports, if possible.
2811         if args == []:
2812             if self.hasOrigin:
2813                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2814
2815             # branches holds mapping from branch name to sha1
2816             branches = p4BranchesInGit(self.importIntoRemotes)
2817
2818             # restrict to just this one, disabling detect-branches
2819             if branch_arg_given:
2820                 short = self.branch.split("/")[-1]
2821                 if short in branches:
2822                     self.p4BranchesInGit = [ short ]
2823             else:
2824                 self.p4BranchesInGit = branches.keys()
2825
2826             if len(self.p4BranchesInGit) > 1:
2827                 if not self.silent:
2828                     print "Importing from/into multiple branches"
2829                 self.detectBranches = True
2830                 for branch in branches.keys():
2831                     self.initialParents[self.refPrefix + branch] = \
2832                         branches[branch]
2833
2834             if self.verbose:
2835                 print "branches: %s" % self.p4BranchesInGit
2836
2837             p4Change = 0
2838             for branch in self.p4BranchesInGit:
2839                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2840
2841                 settings = extractSettingsGitLog(logMsg)
2842
2843                 self.readOptions(settings)
2844                 if (settings.has_key('depot-paths')
2845                     and settings.has_key ('change')):
2846                     change = int(settings['change']) + 1
2847                     p4Change = max(p4Change, change)
2848
2849                     depotPaths = sorted(settings['depot-paths'])
2850                     if self.previousDepotPaths == []:
2851                         self.previousDepotPaths = depotPaths
2852                     else:
2853                         paths = []
2854                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2855                             prev_list = prev.split("/")
2856                             cur_list = cur.split("/")
2857                             for i in range(0, min(len(cur_list), len(prev_list))):
2858                                 if cur_list[i] <> prev_list[i]:
2859                                     i = i - 1
2860                                     break
2861
2862                             paths.append ("/".join(cur_list[:i + 1]))
2863
2864                         self.previousDepotPaths = paths
2865
2866             if p4Change > 0:
2867                 self.depotPaths = sorted(self.previousDepotPaths)
2868                 self.changeRange = "@%s,#head" % p4Change
2869                 if not self.silent and not self.detectBranches:
2870                     print "Performing incremental import into %s git branch" % self.branch
2871
2872         # accept multiple ref name abbreviations:
2873         #    refs/foo/bar/branch -> use it exactly
2874         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2875         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2876         if not self.branch.startswith("refs/"):
2877             if self.importIntoRemotes:
2878                 prepend = "refs/remotes/"
2879             else:
2880                 prepend = "refs/heads/"
2881             if not self.branch.startswith("p4/"):
2882                 prepend += "p4/"
2883             self.branch = prepend + self.branch
2884
2885         if len(args) == 0 and self.depotPaths:
2886             if not self.silent:
2887                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2888         else:
2889             if self.depotPaths and self.depotPaths != args:
2890                 print ("previous import used depot path %s and now %s was specified. "
2891                        "This doesn't work!" % (' '.join (self.depotPaths),
2892                                                ' '.join (args)))
2893                 sys.exit(1)
2894
2895             self.depotPaths = sorted(args)
2896
2897         revision = ""
2898         self.users = {}
2899
2900         # Make sure no revision specifiers are used when --changesfile
2901         # is specified.
2902         bad_changesfile = False
2903         if len(self.changesFile) > 0:
2904             for p in self.depotPaths:
2905                 if p.find("@") >= 0 or p.find("#") >= 0:
2906                     bad_changesfile = True
2907                     break
2908         if bad_changesfile:
2909             die("Option --changesfile is incompatible with revision specifiers")
2910
2911         newPaths = []
2912         for p in self.depotPaths:
2913             if p.find("@") != -1:
2914                 atIdx = p.index("@")
2915                 self.changeRange = p[atIdx:]
2916                 if self.changeRange == "@all":
2917                     self.changeRange = ""
2918                 elif ',' not in self.changeRange:
2919                     revision = self.changeRange
2920                     self.changeRange = ""
2921                 p = p[:atIdx]
2922             elif p.find("#") != -1:
2923                 hashIdx = p.index("#")
2924                 revision = p[hashIdx:]
2925                 p = p[:hashIdx]
2926             elif self.previousDepotPaths == []:
2927                 # pay attention to changesfile, if given, else import
2928                 # the entire p4 tree at the head revision
2929                 if len(self.changesFile) == 0:
2930                     revision = "#head"
2931
2932             p = re.sub ("\.\.\.$", "", p)
2933             if not p.endswith("/"):
2934                 p += "/"
2935
2936             newPaths.append(p)
2937
2938         self.depotPaths = newPaths
2939
2940         # --detect-branches may change this for each branch
2941         self.branchPrefixes = self.depotPaths
2942
2943         self.loadUserMapFromCache()
2944         self.labels = {}
2945         if self.detectLabels:
2946             self.getLabels();
2947
2948         if self.detectBranches:
2949             ## FIXME - what's a P4 projectName ?
2950             self.projectName = self.guessProjectName()
2951
2952             if self.hasOrigin:
2953                 self.getBranchMappingFromGitBranches()
2954             else:
2955                 self.getBranchMapping()
2956             if self.verbose:
2957                 print "p4-git branches: %s" % self.p4BranchesInGit
2958                 print "initial parents: %s" % self.initialParents
2959             for b in self.p4BranchesInGit:
2960                 if b != "master":
2961
2962                     ## FIXME
2963                     b = b[len(self.projectName):]
2964                 self.createdBranches.add(b)
2965
2966         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2967
2968         self.importProcess = subprocess.Popen(["git", "fast-import"],
2969                                               stdin=subprocess.PIPE,
2970                                               stdout=subprocess.PIPE,
2971                                               stderr=subprocess.PIPE);
2972         self.gitOutput = self.importProcess.stdout
2973         self.gitStream = self.importProcess.stdin
2974         self.gitError = self.importProcess.stderr
2975
2976         if revision:
2977             self.importHeadRevision(revision)
2978         else:
2979             changes = []
2980
2981             if len(self.changesFile) > 0:
2982                 output = open(self.changesFile).readlines()
2983                 changeSet = set()
2984                 for line in output:
2985                     changeSet.add(int(line))
2986
2987                 for change in changeSet:
2988                     changes.append(change)
2989
2990                 changes.sort()
2991             else:
2992                 # catch "git p4 sync" with no new branches, in a repo that
2993                 # does not have any existing p4 branches
2994                 if len(args) == 0:
2995                     if not self.p4BranchesInGit:
2996                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
2997
2998                     # The default branch is master, unless --branch is used to
2999                     # specify something else.  Make sure it exists, or complain
3000                     # nicely about how to use --branch.
3001                     if not self.detectBranches:
3002                         if not branch_exists(self.branch):
3003                             if branch_arg_given:
3004                                 die("Error: branch %s does not exist." % self.branch)
3005                             else:
3006                                 die("Error: no branch %s; perhaps specify one with --branch." %
3007                                     self.branch)
3008
3009                 if self.verbose:
3010                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3011                                                               self.changeRange)
3012                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
3013
3014                 if len(self.maxChanges) > 0:
3015                     changes = changes[:min(int(self.maxChanges), len(changes))]
3016
3017             if len(changes) == 0:
3018                 if not self.silent:
3019                     print "No changes to import!"
3020             else:
3021                 if not self.silent and not self.detectBranches:
3022                     print "Import destination: %s" % self.branch
3023
3024                 self.updatedBranches = set()
3025
3026                 if not self.detectBranches:
3027                     if args:
3028                         # start a new branch
3029                         self.initialParent = ""
3030                     else:
3031                         # build on a previous revision
3032                         self.initialParent = parseRevision(self.branch)
3033
3034                 self.importChanges(changes)
3035
3036                 if not self.silent:
3037                     print ""
3038                     if len(self.updatedBranches) > 0:
3039                         sys.stdout.write("Updated branches: ")
3040                         for b in self.updatedBranches:
3041                             sys.stdout.write("%s " % b)
3042                         sys.stdout.write("\n")
3043
3044         if gitConfig("git-p4.importLabels", "--bool") == "true":
3045             self.importLabels = True
3046
3047         if self.importLabels:
3048             p4Labels = getP4Labels(self.depotPaths)
3049             gitTags = getGitTags()
3050
3051             missingP4Labels = p4Labels - gitTags
3052             self.importP4Labels(self.gitStream, missingP4Labels)
3053
3054         self.gitStream.close()
3055         if self.importProcess.wait() != 0:
3056             die("fast-import failed: %s" % self.gitError.read())
3057         self.gitOutput.close()
3058         self.gitError.close()
3059
3060         # Cleanup temporary branches created during import
3061         if self.tempBranches != []:
3062             for branch in self.tempBranches:
3063                 read_pipe("git update-ref -d %s" % branch)
3064             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3065
3066         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3067         # a convenient shortcut refname "p4".
3068         if self.importIntoRemotes:
3069             head_ref = self.refPrefix + "HEAD"
3070             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3071                 system(["git", "symbolic-ref", head_ref, self.branch])
3072
3073         return True
3074
3075 class P4Rebase(Command):
3076     def __init__(self):
3077         Command.__init__(self)
3078         self.options = [
3079                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3080         ]
3081         self.importLabels = False
3082         self.description = ("Fetches the latest revision from perforce and "
3083                             + "rebases the current work (branch) against it")
3084
3085     def run(self, args):
3086         sync = P4Sync()
3087         sync.importLabels = self.importLabels
3088         sync.run([])
3089
3090         return self.rebase()
3091
3092     def rebase(self):
3093         if os.system("git update-index --refresh") != 0:
3094             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3095         if len(read_pipe("git diff-index HEAD --")) > 0:
3096             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3097
3098         [upstream, settings] = findUpstreamBranchPoint()
3099         if len(upstream) == 0:
3100             die("Cannot find upstream branchpoint for rebase")
3101
3102         # the branchpoint may be p4/foo~3, so strip off the parent
3103         upstream = re.sub("~[0-9]+$", "", upstream)
3104
3105         print "Rebasing the current branch onto %s" % upstream
3106         oldHead = read_pipe("git rev-parse HEAD").strip()
3107         system("git rebase %s" % upstream)
3108         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3109         return True
3110
3111 class P4Clone(P4Sync):
3112     def __init__(self):
3113         P4Sync.__init__(self)
3114         self.description = "Creates a new git repository and imports from Perforce into it"
3115         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3116         self.options += [
3117             optparse.make_option("--destination", dest="cloneDestination",
3118                                  action='store', default=None,
3119                                  help="where to leave result of the clone"),
3120             optparse.make_option("-/", dest="cloneExclude",
3121                                  action="append", type="string",
3122                                  help="exclude depot path"),
3123             optparse.make_option("--bare", dest="cloneBare",
3124                                  action="store_true", default=False),
3125         ]
3126         self.cloneDestination = None
3127         self.needsGit = False
3128         self.cloneBare = False
3129
3130     # This is required for the "append" cloneExclude action
3131     def ensure_value(self, attr, value):
3132         if not hasattr(self, attr) or getattr(self, attr) is None:
3133             setattr(self, attr, value)
3134         return getattr(self, attr)
3135
3136     def defaultDestination(self, args):
3137         ## TODO: use common prefix of args?
3138         depotPath = args[0]
3139         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3140         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3141         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3142         depotDir = re.sub(r"/$", "", depotDir)
3143         return os.path.split(depotDir)[1]
3144
3145     def run(self, args):
3146         if len(args) < 1:
3147             return False
3148
3149         if self.keepRepoPath and not self.cloneDestination:
3150             sys.stderr.write("Must specify destination for --keep-path\n")
3151             sys.exit(1)
3152
3153         depotPaths = args
3154
3155         if not self.cloneDestination and len(depotPaths) > 1:
3156             self.cloneDestination = depotPaths[-1]
3157             depotPaths = depotPaths[:-1]
3158
3159         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3160         for p in depotPaths:
3161             if not p.startswith("//"):
3162                 return False
3163
3164         if not self.cloneDestination:
3165             self.cloneDestination = self.defaultDestination(args)
3166
3167         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3168
3169         if not os.path.exists(self.cloneDestination):
3170             os.makedirs(self.cloneDestination)
3171         chdir(self.cloneDestination)
3172
3173         init_cmd = [ "git", "init" ]
3174         if self.cloneBare:
3175             init_cmd.append("--bare")
3176         subprocess.check_call(init_cmd)
3177
3178         if not P4Sync.run(self, depotPaths):
3179             return False
3180
3181         # create a master branch and check out a work tree
3182         if gitBranchExists(self.branch):
3183             system([ "git", "branch", "master", self.branch ])
3184             if not self.cloneBare:
3185                 system([ "git", "checkout", "-f" ])
3186         else:
3187             print 'Not checking out any branch, use ' \
3188                   '"git checkout -q -b master <branch>"'
3189
3190         # auto-set this variable if invoked with --use-client-spec
3191         if self.useClientSpec_from_options:
3192             system("git config --bool git-p4.useclientspec true")
3193
3194         return True
3195
3196 class P4Branches(Command):
3197     def __init__(self):
3198         Command.__init__(self)
3199         self.options = [ ]
3200         self.description = ("Shows the git branches that hold imports and their "
3201                             + "corresponding perforce depot paths")
3202         self.verbose = False
3203
3204     def run(self, args):
3205         if originP4BranchesExist():
3206             createOrUpdateBranchesFromOrigin()
3207
3208         cmdline = "git rev-parse --symbolic "
3209         cmdline += " --remotes"
3210
3211         for line in read_pipe_lines(cmdline):
3212             line = line.strip()
3213
3214             if not line.startswith('p4/') or line == "p4/HEAD":
3215                 continue
3216             branch = line
3217
3218             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3219             settings = extractSettingsGitLog(log)
3220
3221             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3222         return True
3223
3224 class HelpFormatter(optparse.IndentedHelpFormatter):
3225     def __init__(self):
3226         optparse.IndentedHelpFormatter.__init__(self)
3227
3228     def format_description(self, description):
3229         if description:
3230             return description + "\n"
3231         else:
3232             return ""
3233
3234 def printUsage(commands):
3235     print "usage: %s <command> [options]" % sys.argv[0]
3236     print ""
3237     print "valid commands: %s" % ", ".join(commands)
3238     print ""
3239     print "Try %s <command> --help for command specific help." % sys.argv[0]
3240     print ""
3241
3242 commands = {
3243     "debug" : P4Debug,
3244     "submit" : P4Submit,
3245     "commit" : P4Submit,
3246     "sync" : P4Sync,
3247     "rebase" : P4Rebase,
3248     "clone" : P4Clone,
3249     "rollback" : P4RollBack,
3250     "branches" : P4Branches
3251 }
3252
3253
3254 def main():
3255     if len(sys.argv[1:]) == 0:
3256         printUsage(commands.keys())
3257         sys.exit(2)
3258
3259     cmdName = sys.argv[1]
3260     try:
3261         klass = commands[cmdName]
3262         cmd = klass()
3263     except KeyError:
3264         print "unknown command %s" % cmdName
3265         print ""
3266         printUsage(commands.keys())
3267         sys.exit(2)
3268
3269     options = cmd.options
3270     cmd.gitdir = os.environ.get("GIT_DIR", None)
3271
3272     args = sys.argv[2:]
3273
3274     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3275     if cmd.needsGit:
3276         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3277
3278     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3279                                    options,
3280                                    description = cmd.description,
3281                                    formatter = HelpFormatter())
3282
3283     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3284     global verbose
3285     verbose = cmd.verbose
3286     if cmd.needsGit:
3287         if cmd.gitdir == None:
3288             cmd.gitdir = os.path.abspath(".git")
3289             if not isValidGitDir(cmd.gitdir):
3290                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3291                 if os.path.exists(cmd.gitdir):
3292                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3293                     if len(cdup) > 0:
3294                         chdir(cdup);
3295
3296         if not isValidGitDir(cmd.gitdir):
3297             if isValidGitDir(cmd.gitdir + "/.git"):
3298                 cmd.gitdir += "/.git"
3299             else:
3300                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3301
3302         os.environ["GIT_DIR"] = cmd.gitdir
3303
3304     if not cmd.run(args):
3305         parser.print_help()
3306         sys.exit(2)
3307
3308
3309 if __name__ == '__main__':
3310     main()