git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import sys
  12 if sys.hexversion < 0x02040000:
  13     # The limiter is the subprocess module
  14     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  15     sys.exit(1)
  16
  17 import optparse, os, marshal, subprocess, shelve
  18 import tempfile, getopt, os.path, time, platform
  19 import re, shutil
  20
  21 verbose = False
  22
  23 # Only labels/tags matching this will be imported/exported
  24 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  25
  26 def p4_build_cmd(cmd):
  27     """Build a suitable p4 command line.
  28
  29     This consolidates building and returning a p4 command line into one
  30     location. It means that hooking into the environment, or other configuration
  31     can be done more easily.
  32     """
  33     real_cmd = ["p4"]
  34
  35     user = gitConfig("git-p4.user")
  36     if len(user) > 0:
  37         real_cmd += ["-u",user]
  38
  39     password = gitConfig("git-p4.password")
  40     if len(password) > 0:
  41         real_cmd += ["-P", password]
  42
  43     port = gitConfig("git-p4.port")
  44     if len(port) > 0:
  45         real_cmd += ["-p", port]
  46
  47     host = gitConfig("git-p4.host")
  48     if len(host) > 0:
  49         real_cmd += ["-H", host]
  50
  51     client = gitConfig("git-p4.client")
  52     if len(client) > 0:
  53         real_cmd += ["-c", client]
  54
  55
  56     if isinstance(cmd,basestring):
  57         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  58     else:
  59         real_cmd += cmd
  60     return real_cmd
  61
  62 def chdir(dir):
  63     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  64     # not using the shell, we have to set it ourselves.  This path could
  65     # be relative, so go there first, then figure out where we ended up.
  66     os.chdir(dir)
  67     os.environ['PWD'] = os.getcwd()
  68
  69 def die(msg):
  70     if verbose:
  71         raise Exception(msg)
  72     else:
  73         sys.stderr.write(msg + "\n")
  74         sys.exit(1)
  75
  76 def write_pipe(c, stdin):
  77     if verbose:
  78         sys.stderr.write('Writing pipe: %s\n' % str(c))
  79
  80     expand = isinstance(c,basestring)
  81     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  82     pipe = p.stdin
  83     val = pipe.write(stdin)
  84     pipe.close()
  85     if p.wait():
  86         die('Command failed: %s' % str(c))
  87
  88     return val
  89
  90 def p4_write_pipe(c, stdin):
  91     real_cmd = p4_build_cmd(c)
  92     return write_pipe(real_cmd, stdin)
  93
  94 def read_pipe(c, ignore_error=False):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % str(c))
  97
  98     expand = isinstance(c,basestring)
  99     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 100     pipe = p.stdout
 101     val = pipe.read()
 102     if p.wait() and not ignore_error:
 103         die('Command failed: %s' % str(c))
 104
 105     return val
 106
 107 def p4_read_pipe(c, ignore_error=False):
 108     real_cmd = p4_build_cmd(c)
 109     return read_pipe(real_cmd, ignore_error)
 110
 111 def read_pipe_lines(c):
 112     if verbose:
 113         sys.stderr.write('Reading pipe: %s\n' % str(c))
 114
 115     expand = isinstance(c, basestring)
 116     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 117     pipe = p.stdout
 118     val = pipe.readlines()
 119     if pipe.close() or p.wait():
 120         die('Command failed: %s' % str(c))
 121
 122     return val
 123
 124 def p4_read_pipe_lines(c):
 125     """Specifically invoke p4 on the command supplied. """
 126     real_cmd = p4_build_cmd(c)
 127     return read_pipe_lines(real_cmd)
 128
 129 def p4_has_command(cmd):
 130     """Ask p4 for help on this command.  If it returns an error, the
 131        command does not exist in this version of p4."""
 132     real_cmd = p4_build_cmd(["help", cmd])
 133     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 134                                    stderr=subprocess.PIPE)
 135     p.communicate()
 136     return p.returncode == 0
 137
 138 def p4_has_move_command():
 139     """See if the move command exists, that it supports -k, and that
 140        it has not been administratively disabled.  The arguments
 141        must be correct, but the filenames do not have to exist.  Use
 142        ones with wildcards so even if they exist, it will fail."""
 143
 144     if not p4_has_command("move"):
 145         return False
 146     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 147     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 148     (out, err) = p.communicate()
 149     # return code will be 1 in either case
 150     if err.find("Invalid option") >= 0:
 151         return False
 152     if err.find("disabled") >= 0:
 153         return False
 154     # assume it failed because @... was invalid changelist
 155     return True
 156
 157 def system(cmd):
 158     expand = isinstance(cmd,basestring)
 159     if verbose:
 160         sys.stderr.write("executing %s\n" % str(cmd))
 161     subprocess.check_call(cmd, shell=expand)
 162
 163 def p4_system(cmd):
 164     """Specifically invoke p4 as the system command. """
 165     real_cmd = p4_build_cmd(cmd)
 166     expand = isinstance(real_cmd, basestring)
 167     subprocess.check_call(real_cmd, shell=expand)
 168
 169 def p4_integrate(src, dest):
 170     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 171
 172 def p4_sync(f, *options):
 173     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 174
 175 def p4_add(f):
 176     # forcibly add file names with wildcards
 177     if wildcard_present(f):
 178         p4_system(["add", "-f", f])
 179     else:
 180         p4_system(["add", f])
 181
 182 def p4_delete(f):
 183     p4_system(["delete", wildcard_encode(f)])
 184
 185 def p4_edit(f):
 186     p4_system(["edit", wildcard_encode(f)])
 187
 188 def p4_revert(f):
 189     p4_system(["revert", wildcard_encode(f)])
 190
 191 def p4_reopen(type, f):
 192     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 193
 194 def p4_move(src, dest):
 195     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 196
 197 def p4_describe(change):
 198     """Make sure it returns a valid result by checking for
 199        the presence of field "time".  Return a dict of the
 200        results."""
 201
 202     ds = p4CmdList(["describe", "-s", str(change)])
 203     if len(ds) != 1:
 204         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 205
 206     d = ds[0]
 207
 208     if "p4ExitCode" in d:
 209         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 210                                                       str(d)))
 211     if "code" in d:
 212         if d["code"] == "error":
 213             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 214
 215     if "time" not in d:
 216         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 217
 218     return d
 219
 220 #
 221 # Canonicalize the p4 type and return a tuple of the
 222 # base type, plus any modifiers.  See "p4 help filetypes"
 223 # for a list and explanation.
 224 #
 225 def split_p4_type(p4type):
 226
 227     p4_filetypes_historical = {
 228         "ctempobj": "binary+Sw",
 229         "ctext": "text+C",
 230         "cxtext": "text+Cx",
 231         "ktext": "text+k",
 232         "kxtext": "text+kx",
 233         "ltext": "text+F",
 234         "tempobj": "binary+FSw",
 235         "ubinary": "binary+F",
 236         "uresource": "resource+F",
 237         "uxbinary": "binary+Fx",
 238         "xbinary": "binary+x",
 239         "xltext": "text+Fx",
 240         "xtempobj": "binary+Swx",
 241         "xtext": "text+x",
 242         "xunicode": "unicode+x",
 243         "xutf16": "utf16+x",
 244     }
 245     if p4type in p4_filetypes_historical:
 246         p4type = p4_filetypes_historical[p4type]
 247     mods = ""
 248     s = p4type.split("+")
 249     base = s[0]
 250     mods = ""
 251     if len(s) > 1:
 252         mods = s[1]
 253     return (base, mods)
 254
 255 #
 256 # return the raw p4 type of a file (text, text+ko, etc)
 257 #
 258 def p4_type(file):
 259     results = p4CmdList(["fstat", "-T", "headType", file])
 260     return results[0]['headType']
 261
 262 #
 263 # Given a type base and modifier, return a regexp matching
 264 # the keywords that can be expanded in the file
 265 #
 266 def p4_keywords_regexp_for_type(base, type_mods):
 267     if base in ("text", "unicode", "binary"):
 268         kwords = None
 269         if "ko" in type_mods:
 270             kwords = 'Id|Header'
 271         elif "k" in type_mods:
 272             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 273         else:
 274             return None
 275         pattern = r"""
 276             \$              # Starts with a dollar, followed by...
 277             (%s)            # one of the keywords, followed by...
 278             (:[^$\n]+)?     # possibly an old expansion, followed by...
 279             \$              # another dollar
 280             """ % kwords
 281         return pattern
 282     else:
 283         return None
 284
 285 #
 286 # Given a file, return a regexp matching the possible
 287 # RCS keywords that will be expanded, or None for files
 288 # with kw expansion turned off.
 289 #
 290 def p4_keywords_regexp_for_file(file):
 291     if not os.path.exists(file):
 292         return None
 293     else:
 294         (type_base, type_mods) = split_p4_type(p4_type(file))
 295         return p4_keywords_regexp_for_type(type_base, type_mods)
 296
 297 def setP4ExecBit(file, mode):
 298     # Reopens an already open file and changes the execute bit to match
 299     # the execute bit setting in the passed in mode.
 300
 301     p4Type = "+x"
 302
 303     if not isModeExec(mode):
 304         p4Type = getP4OpenedType(file)
 305         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 306         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 307         if p4Type[-1] == "+":
 308             p4Type = p4Type[0:-1]
 309
 310     p4_reopen(p4Type, file)
 311
 312 def getP4OpenedType(file):
 313     # Returns the perforce file type for the given file.
 314
 315     result = p4_read_pipe(["opened", wildcard_encode(file)])
 316     match = re.match(".*\((.+)\)\r?$", result)
 317     if match:
 318         return match.group(1)
 319     else:
 320         die("Could not determine file type for %s (result: '%s')" % (file, result))
 321
 322 # Return the set of all p4 labels
 323 def getP4Labels(depotPaths):
 324     labels = set()
 325     if isinstance(depotPaths,basestring):
 326         depotPaths = [depotPaths]
 327
 328     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 329         label = l['label']
 330         labels.add(label)
 331
 332     return labels
 333
 334 # Return the set of all git tags
 335 def getGitTags():
 336     gitTags = set()
 337     for line in read_pipe_lines(["git", "tag"]):
 338         tag = line.strip()
 339         gitTags.add(tag)
 340     return gitTags
 341
 342 def diffTreePattern():
 343     # This is a simple generator for the diff tree regex pattern. This could be
 344     # a class variable if this and parseDiffTreeEntry were a part of a class.
 345     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 346     while True:
 347         yield pattern
 348
 349 def parseDiffTreeEntry(entry):
 350     """Parses a single diff tree entry into its component elements.
 351
 352     See git-diff-tree(1) manpage for details about the format of the diff
 353     output. This method returns a dictionary with the following elements:
 354
 355     src_mode - The mode of the source file
 356     dst_mode - The mode of the destination file
 357     src_sha1 - The sha1 for the source file
 358     dst_sha1 - The sha1 fr the destination file
 359     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 360     status_score - The score for the status (applicable for 'C' and 'R'
 361                    statuses). This is None if there is no score.
 362     src - The path for the source file.
 363     dst - The path for the destination file. This is only present for
 364           copy or renames. If it is not present, this is None.
 365
 366     If the pattern is not matched, None is returned."""
 367
 368     match = diffTreePattern().next().match(entry)
 369     if match:
 370         return {
 371             'src_mode': match.group(1),
 372             'dst_mode': match.group(2),
 373             'src_sha1': match.group(3),
 374             'dst_sha1': match.group(4),
 375             'status': match.group(5),
 376             'status_score': match.group(6),
 377             'src': match.group(7),
 378             'dst': match.group(10)
 379         }
 380     return None
 381
 382 def isModeExec(mode):
 383     # Returns True if the given git mode represents an executable file,
 384     # otherwise False.
 385     return mode[-3:] == "755"
 386
 387 def isModeExecChanged(src_mode, dst_mode):
 388     return isModeExec(src_mode) != isModeExec(dst_mode)
 389
 390 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 391
 392     if isinstance(cmd,basestring):
 393         cmd = "-G " + cmd
 394         expand = True
 395     else:
 396         cmd = ["-G"] + cmd
 397         expand = False
 398
 399     cmd = p4_build_cmd(cmd)
 400     if verbose:
 401         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 402
 403     # Use a temporary file to avoid deadlocks without
 404     # subprocess.communicate(), which would put another copy
 405     # of stdout into memory.
 406     stdin_file = None
 407     if stdin is not None:
 408         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 409         if isinstance(stdin,basestring):
 410             stdin_file.write(stdin)
 411         else:
 412             for i in stdin:
 413                 stdin_file.write(i + '\n')
 414         stdin_file.flush()
 415         stdin_file.seek(0)
 416
 417     p4 = subprocess.Popen(cmd,
 418                           shell=expand,
 419                           stdin=stdin_file,
 420                           stdout=subprocess.PIPE)
 421
 422     result = []
 423     try:
 424         while True:
 425             entry = marshal.load(p4.stdout)
 426             if cb is not None:
 427                 cb(entry)
 428             else:
 429                 result.append(entry)
 430     except EOFError:
 431         pass
 432     exitCode = p4.wait()
 433     if exitCode != 0:
 434         entry = {}
 435         entry["p4ExitCode"] = exitCode
 436         result.append(entry)
 437
 438     return result
 439
 440 def p4Cmd(cmd):
 441     list = p4CmdList(cmd)
 442     result = {}
 443     for entry in list:
 444         result.update(entry)
 445     return result;
 446
 447 def p4Where(depotPath):
 448     if not depotPath.endswith("/"):
 449         depotPath += "/"
 450     depotPath = depotPath + "..."
 451     outputList = p4CmdList(["where", depotPath])
 452     output = None
 453     for entry in outputList:
 454         if "depotFile" in entry:
 455             if entry["depotFile"] == depotPath:
 456                 output = entry
 457                 break
 458         elif "data" in entry:
 459             data = entry.get("data")
 460             space = data.find(" ")
 461             if data[:space] == depotPath:
 462                 output = entry
 463                 break
 464     if output == None:
 465         return ""
 466     if output["code"] == "error":
 467         return ""
 468     clientPath = ""
 469     if "path" in output:
 470         clientPath = output.get("path")
 471     elif "data" in output:
 472         data = output.get("data")
 473         lastSpace = data.rfind(" ")
 474         clientPath = data[lastSpace + 1:]
 475
 476     if clientPath.endswith("..."):
 477         clientPath = clientPath[:-3]
 478     return clientPath
 479
 480 def currentGitBranch():
 481     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 482
 483 def isValidGitDir(path):
 484     if (os.path.exists(path + "/HEAD")
 485         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 486         return True;
 487     return False
 488
 489 def parseRevision(ref):
 490     return read_pipe("git rev-parse %s" % ref).strip()
 491
 492 def branchExists(ref):
 493     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 494                      ignore_error=True)
 495     return len(rev) > 0
 496
 497 def extractLogMessageFromGitCommit(commit):
 498     logMessage = ""
 499
 500     ## fixme: title is first line of commit, not 1st paragraph.
 501     foundTitle = False
 502     for log in read_pipe_lines("git cat-file commit %s" % commit):
 503        if not foundTitle:
 504            if len(log) == 1:
 505                foundTitle = True
 506            continue
 507
 508        logMessage += log
 509     return logMessage
 510
 511 def extractSettingsGitLog(log):
 512     values = {}
 513     for line in log.split("\n"):
 514         line = line.strip()
 515         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 516         if not m:
 517             continue
 518
 519         assignments = m.group(1).split (':')
 520         for a in assignments:
 521             vals = a.split ('=')
 522             key = vals[0].strip()
 523             val = ('='.join (vals[1:])).strip()
 524             if val.endswith ('\"') and val.startswith('"'):
 525                 val = val[1:-1]
 526
 527             values[key] = val
 528
 529     paths = values.get("depot-paths")
 530     if not paths:
 531         paths = values.get("depot-path")
 532     if paths:
 533         values['depot-paths'] = paths.split(',')
 534     return values
 535
 536 def gitBranchExists(branch):
 537     proc = subprocess.Popen(["git", "rev-parse", branch],
 538                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 539     return proc.wait() == 0;
 540
 541 _gitConfig = {}
 542 def gitConfig(key, args = None): # set args to "--bool", for instance
 543     if not _gitConfig.has_key(key):
 544         argsFilter = ""
 545         if args != None:
 546             argsFilter = "%s " % args
 547         cmd = "git config %s%s" % (argsFilter, key)
 548         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 549     return _gitConfig[key]
 550
 551 def gitConfigList(key):
 552     if not _gitConfig.has_key(key):
 553         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 554     return _gitConfig[key]
 555
 556 def p4BranchesInGit(branchesAreInRemotes = True):
 557     branches = {}
 558
 559     cmdline = "git rev-parse --symbolic "
 560     if branchesAreInRemotes:
 561         cmdline += " --remotes"
 562     else:
 563         cmdline += " --branches"
 564
 565     for line in read_pipe_lines(cmdline):
 566         line = line.strip()
 567
 568         ## only import to p4/
 569         if not line.startswith('p4/') or line == "p4/HEAD":
 570             continue
 571         branch = line
 572
 573         # strip off p4
 574         branch = re.sub ("^p4/", "", line)
 575
 576         branches[branch] = parseRevision(line)
 577     return branches
 578
 579 def findUpstreamBranchPoint(head = "HEAD"):
 580     branches = p4BranchesInGit()
 581     # map from depot-path to branch name
 582     branchByDepotPath = {}
 583     for branch in branches.keys():
 584         tip = branches[branch]
 585         log = extractLogMessageFromGitCommit(tip)
 586         settings = extractSettingsGitLog(log)
 587         if settings.has_key("depot-paths"):
 588             paths = ",".join(settings["depot-paths"])
 589             branchByDepotPath[paths] = "remotes/p4/" + branch
 590
 591     settings = None
 592     parent = 0
 593     while parent < 65535:
 594         commit = head + "~%s" % parent
 595         log = extractLogMessageFromGitCommit(commit)
 596         settings = extractSettingsGitLog(log)
 597         if settings.has_key("depot-paths"):
 598             paths = ",".join(settings["depot-paths"])
 599             if branchByDepotPath.has_key(paths):
 600                 return [branchByDepotPath[paths], settings]
 601
 602         parent = parent + 1
 603
 604     return ["", settings]
 605
 606 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 607     if not silent:
 608         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 609                % localRefPrefix)
 610
 611     originPrefix = "origin/p4/"
 612
 613     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 614         line = line.strip()
 615         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 616             continue
 617
 618         headName = line[len(originPrefix):]
 619         remoteHead = localRefPrefix + headName
 620         originHead = line
 621
 622         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 623         if (not original.has_key('depot-paths')
 624             or not original.has_key('change')):
 625             continue
 626
 627         update = False
 628         if not gitBranchExists(remoteHead):
 629             if verbose:
 630                 print "creating %s" % remoteHead
 631             update = True
 632         else:
 633             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 634             if settings.has_key('change') > 0:
 635                 if settings['depot-paths'] == original['depot-paths']:
 636                     originP4Change = int(original['change'])
 637                     p4Change = int(settings['change'])
 638                     if originP4Change > p4Change:
 639                         print ("%s (%s) is newer than %s (%s). "
 640                                "Updating p4 branch from origin."
 641                                % (originHead, originP4Change,
 642                                   remoteHead, p4Change))
 643                         update = True
 644                 else:
 645                     print ("Ignoring: %s was imported from %s while "
 646                            "%s was imported from %s"
 647                            % (originHead, ','.join(original['depot-paths']),
 648                               remoteHead, ','.join(settings['depot-paths'])))
 649
 650         if update:
 651             system("git update-ref %s %s" % (remoteHead, originHead))
 652
 653 def originP4BranchesExist():
 654         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 655
 656 def p4ChangesForPaths(depotPaths, changeRange):
 657     assert depotPaths
 658     cmd = ['changes']
 659     for p in depotPaths:
 660         cmd += ["%s...%s" % (p, changeRange)]
 661     output = p4_read_pipe_lines(cmd)
 662
 663     changes = {}
 664     for line in output:
 665         changeNum = int(line.split(" ")[1])
 666         changes[changeNum] = True
 667
 668     changelist = changes.keys()
 669     changelist.sort()
 670     return changelist
 671
 672 def p4PathStartsWith(path, prefix):
 673     # This method tries to remedy a potential mixed-case issue:
 674     #
 675     # If UserA adds  //depot/DirA/file1
 676     # and UserB adds //depot/dira/file2
 677     #
 678     # we may or may not have a problem. If you have core.ignorecase=true,
 679     # we treat DirA and dira as the same directory
 680     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 681     if ignorecase:
 682         return path.lower().startswith(prefix.lower())
 683     return path.startswith(prefix)
 684
 685 def getClientSpec():
 686     """Look at the p4 client spec, create a View() object that contains
 687        all the mappings, and return it."""
 688
 689     specList = p4CmdList("client -o")
 690     if len(specList) != 1:
 691         die('Output from "client -o" is %d lines, expecting 1' %
 692             len(specList))
 693
 694     # dictionary of all client parameters
 695     entry = specList[0]
 696
 697     # just the keys that start with "View"
 698     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 699
 700     # hold this new View
 701     view = View()
 702
 703     # append the lines, in order, to the view
 704     for view_num in range(len(view_keys)):
 705         k = "View%d" % view_num
 706         if k not in view_keys:
 707             die("Expected view key %s missing" % k)
 708         view.append(entry[k])
 709
 710     return view
 711
 712 def getClientRoot():
 713     """Grab the client directory."""
 714
 715     output = p4CmdList("client -o")
 716     if len(output) != 1:
 717         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 718
 719     entry = output[0]
 720     if "Root" not in entry:
 721         die('Client has no "Root"')
 722
 723     return entry["Root"]
 724
 725 #
 726 # P4 wildcards are not allowed in filenames.  P4 complains
 727 # if you simply add them, but you can force it with "-f", in
 728 # which case it translates them into %xx encoding internally.
 729 #
 730 def wildcard_decode(path):
 731     # Search for and fix just these four characters.  Do % last so
 732     # that fixing it does not inadvertently create new %-escapes.
 733     # Cannot have * in a filename in windows; untested as to
 734     # what p4 would do in such a case.
 735     if not platform.system() == "Windows":
 736         path = path.replace("%2A", "*")
 737     path = path.replace("%23", "#") \
 738                .replace("%40", "@") \
 739                .replace("%25", "%")
 740     return path
 741
 742 def wildcard_encode(path):
 743     # do % first to avoid double-encoding the %s introduced here
 744     path = path.replace("%", "%25") \
 745                .replace("*", "%2A") \
 746                .replace("#", "%23") \
 747                .replace("@", "%40")
 748     return path
 749
 750 def wildcard_present(path):
 751     return path.translate(None, "*#@%") != path
 752
 753 class Command:
 754     def __init__(self):
 755         self.usage = "usage: %prog [options]"
 756         self.needsGit = True
 757         self.verbose = False
 758
 759 class P4UserMap:
 760     def __init__(self):
 761         self.userMapFromPerforceServer = False
 762         self.myP4UserId = None
 763
 764     def p4UserId(self):
 765         if self.myP4UserId:
 766             return self.myP4UserId
 767
 768         results = p4CmdList("user -o")
 769         for r in results:
 770             if r.has_key('User'):
 771                 self.myP4UserId = r['User']
 772                 return r['User']
 773         die("Could not find your p4 user id")
 774
 775     def p4UserIsMe(self, p4User):
 776         # return True if the given p4 user is actually me
 777         me = self.p4UserId()
 778         if not p4User or p4User != me:
 779             return False
 780         else:
 781             return True
 782
 783     def getUserCacheFilename(self):
 784         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 785         return home + "/.gitp4-usercache.txt"
 786
 787     def getUserMapFromPerforceServer(self):
 788         if self.userMapFromPerforceServer:
 789             return
 790         self.users = {}
 791         self.emails = {}
 792
 793         for output in p4CmdList("users"):
 794             if not output.has_key("User"):
 795                 continue
 796             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 797             self.emails[output["Email"]] = output["User"]
 798
 799
 800         s = ''
 801         for (key, val) in self.users.items():
 802             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 803
 804         open(self.getUserCacheFilename(), "wb").write(s)
 805         self.userMapFromPerforceServer = True
 806
 807     def loadUserMapFromCache(self):
 808         self.users = {}
 809         self.userMapFromPerforceServer = False
 810         try:
 811             cache = open(self.getUserCacheFilename(), "rb")
 812             lines = cache.readlines()
 813             cache.close()
 814             for line in lines:
 815                 entry = line.strip().split("\t")
 816                 self.users[entry[0]] = entry[1]
 817         except IOError:
 818             self.getUserMapFromPerforceServer()
 819
 820 class P4Debug(Command):
 821     def __init__(self):
 822         Command.__init__(self)
 823         self.options = []
 824         self.description = "A tool to debug the output of p4 -G."
 825         self.needsGit = False
 826
 827     def run(self, args):
 828         j = 0
 829         for output in p4CmdList(args):
 830             print 'Element: %d' % j
 831             j += 1
 832             print output
 833         return True
 834
 835 class P4RollBack(Command):
 836     def __init__(self):
 837         Command.__init__(self)
 838         self.options = [
 839             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 840         ]
 841         self.description = "A tool to debug the multi-branch import. Don't use :)"
 842         self.rollbackLocalBranches = False
 843
 844     def run(self, args):
 845         if len(args) != 1:
 846             return False
 847         maxChange = int(args[0])
 848
 849         if "p4ExitCode" in p4Cmd("changes -m 1"):
 850             die("Problems executing p4");
 851
 852         if self.rollbackLocalBranches:
 853             refPrefix = "refs/heads/"
 854             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 855         else:
 856             refPrefix = "refs/remotes/"
 857             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 858
 859         for line in lines:
 860             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 861                 line = line.strip()
 862                 ref = refPrefix + line
 863                 log = extractLogMessageFromGitCommit(ref)
 864                 settings = extractSettingsGitLog(log)
 865
 866                 depotPaths = settings['depot-paths']
 867                 change = settings['change']
 868
 869                 changed = False
 870
 871                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 872                                                            for p in depotPaths]))) == 0:
 873                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 874                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 875                     continue
 876
 877                 while change and int(change) > maxChange:
 878                     changed = True
 879                     if self.verbose:
 880                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 881                     system("git update-ref %s \"%s^\"" % (ref, ref))
 882                     log = extractLogMessageFromGitCommit(ref)
 883                     settings =  extractSettingsGitLog(log)
 884
 885
 886                     depotPaths = settings['depot-paths']
 887                     change = settings['change']
 888
 889                 if changed:
 890                     print "%s rewound to %s" % (ref, change)
 891
 892         return True
 893
 894 class P4Submit(Command, P4UserMap):
 895
 896     conflict_behavior_choices = ("ask", "skip", "quit")
 897
 898     def __init__(self):
 899         Command.__init__(self)
 900         P4UserMap.__init__(self)
 901         self.options = [
 902                 optparse.make_option("--origin", dest="origin"),
 903                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 904                 # preserve the user, requires relevant p4 permissions
 905                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 906                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 907                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
 908                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
 909                 optparse.make_option("--conflict", dest="conflict_behavior",
 910                                      choices=self.conflict_behavior_choices)
 911         ]
 912         self.description = "Submit changes from git to the perforce depot."
 913         self.usage += " [name of git branch to submit into perforce depot]"
 914         self.origin = ""
 915         self.detectRenames = False
 916         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 917         self.dry_run = False
 918         self.prepare_p4_only = False
 919         self.conflict_behavior = None
 920         self.isWindows = (platform.system() == "Windows")
 921         self.exportLabels = False
 922         self.p4HasMoveCommand = p4_has_move_command()
 923
 924     def check(self):
 925         if len(p4CmdList("opened ...")) > 0:
 926             die("You have files opened with perforce! Close them before starting the sync.")
 927
 928     def separate_jobs_from_description(self, message):
 929         """Extract and return a possible Jobs field in the commit
 930            message.  It goes into a separate section in the p4 change
 931            specification.
 932
 933            A jobs line starts with "Jobs:" and looks like a new field
 934            in a form.  Values are white-space separated on the same
 935            line or on following lines that start with a tab.
 936
 937            This does not parse and extract the full git commit message
 938            like a p4 form.  It just sees the Jobs: line as a marker
 939            to pass everything from then on directly into the p4 form,
 940            but outside the description section.
 941
 942            Return a tuple (stripped log message, jobs string)."""
 943
 944         m = re.search(r'^Jobs:', message, re.MULTILINE)
 945         if m is None:
 946             return (message, None)
 947
 948         jobtext = message[m.start():]
 949         stripped_message = message[:m.start()].rstrip()
 950         return (stripped_message, jobtext)
 951
 952     def prepareLogMessage(self, template, message, jobs):
 953         """Edits the template returned from "p4 change -o" to insert
 954            the message in the Description field, and the jobs text in
 955            the Jobs field."""
 956         result = ""
 957
 958         inDescriptionSection = False
 959
 960         for line in template.split("\n"):
 961             if line.startswith("#"):
 962                 result += line + "\n"
 963                 continue
 964
 965             if inDescriptionSection:
 966                 if line.startswith("Files:") or line.startswith("Jobs:"):
 967                     inDescriptionSection = False
 968                     # insert Jobs section
 969                     if jobs:
 970                         result += jobs + "\n"
 971                 else:
 972                     continue
 973             else:
 974                 if line.startswith("Description:"):
 975                     inDescriptionSection = True
 976                     line += "\n"
 977                     for messageLine in message.split("\n"):
 978                         line += "\t" + messageLine + "\n"
 979
 980             result += line + "\n"
 981
 982         return result
 983
 984     def patchRCSKeywords(self, file, pattern):
 985         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
 986         (handle, outFileName) = tempfile.mkstemp(dir='.')
 987         try:
 988             outFile = os.fdopen(handle, "w+")
 989             inFile = open(file, "r")
 990             regexp = re.compile(pattern, re.VERBOSE)
 991             for line in inFile.readlines():
 992                 line = regexp.sub(r'$\1$', line)
 993                 outFile.write(line)
 994             inFile.close()
 995             outFile.close()
 996             # Forcibly overwrite the original file
 997             os.unlink(file)
 998             shutil.move(outFileName, file)
 999         except:
1000             # cleanup our temporary file
1001             os.unlink(outFileName)
1002             print "Failed to strip RCS keywords in %s" % file
1003             raise
1004
1005         print "Patched up RCS keywords in %s" % file
1006
1007     def p4UserForCommit(self,id):
1008         # Return the tuple (perforce user,git email) for a given git commit id
1009         self.getUserMapFromPerforceServer()
1010         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
1011         gitEmail = gitEmail.strip()
1012         if not self.emails.has_key(gitEmail):
1013             return (None,gitEmail)
1014         else:
1015             return (self.emails[gitEmail],gitEmail)
1016
1017     def checkValidP4Users(self,commits):
1018         # check if any git authors cannot be mapped to p4 users
1019         for id in commits:
1020             (user,email) = self.p4UserForCommit(id)
1021             if not user:
1022                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1023                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
1024                     print "%s" % msg
1025                 else:
1026                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1027
1028     def lastP4Changelist(self):
1029         # Get back the last changelist number submitted in this client spec. This
1030         # then gets used to patch up the username in the change. If the same
1031         # client spec is being used by multiple processes then this might go
1032         # wrong.
1033         results = p4CmdList("client -o")        # find the current client
1034         client = None
1035         for r in results:
1036             if r.has_key('Client'):
1037                 client = r['Client']
1038                 break
1039         if not client:
1040             die("could not get client spec")
1041         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1042         for r in results:
1043             if r.has_key('change'):
1044                 return r['change']
1045         die("Could not get changelist number for last submit - cannot patch up user details")
1046
1047     def modifyChangelistUser(self, changelist, newUser):
1048         # fixup the user field of a changelist after it has been submitted.
1049         changes = p4CmdList("change -o %s" % changelist)
1050         if len(changes) != 1:
1051             die("Bad output from p4 change modifying %s to user %s" %
1052                 (changelist, newUser))
1053
1054         c = changes[0]
1055         if c['User'] == newUser: return   # nothing to do
1056         c['User'] = newUser
1057         input = marshal.dumps(c)
1058
1059         result = p4CmdList("change -f -i", stdin=input)
1060         for r in result:
1061             if r.has_key('code'):
1062                 if r['code'] == 'error':
1063                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1064             if r.has_key('data'):
1065                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1066                 return
1067         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1068
1069     def canChangeChangelists(self):
1070         # check to see if we have p4 admin or super-user permissions, either of
1071         # which are required to modify changelists.
1072         results = p4CmdList(["protects", self.depotPath])
1073         for r in results:
1074             if r.has_key('perm'):
1075                 if r['perm'] == 'admin':
1076                     return 1
1077                 if r['perm'] == 'super':
1078                     return 1
1079         return 0
1080
1081     def prepareSubmitTemplate(self):
1082         """Run "p4 change -o" to grab a change specification template.
1083            This does not use "p4 -G", as it is nice to keep the submission
1084            template in original order, since a human might edit it.
1085
1086            Remove lines in the Files section that show changes to files
1087            outside the depot path we're committing into."""
1088
1089         template = ""
1090         inFilesSection = False
1091         for line in p4_read_pipe_lines(['change', '-o']):
1092             if line.endswith("\r\n"):
1093                 line = line[:-2] + "\n"
1094             if inFilesSection:
1095                 if line.startswith("\t"):
1096                     # path starts and ends with a tab
1097                     path = line[1:]
1098                     lastTab = path.rfind("\t")
1099                     if lastTab != -1:
1100                         path = path[:lastTab]
1101                         if not p4PathStartsWith(path, self.depotPath):
1102                             continue
1103                 else:
1104                     inFilesSection = False
1105             else:
1106                 if line.startswith("Files:"):
1107                     inFilesSection = True
1108
1109             template += line
1110
1111         return template
1112
1113     def edit_template(self, template_file):
1114         """Invoke the editor to let the user change the submission
1115            message.  Return true if okay to continue with the submit."""
1116
1117         # if configured to skip the editing part, just submit
1118         if gitConfig("git-p4.skipSubmitEdit") == "true":
1119             return True
1120
1121         # look at the modification time, to check later if the user saved
1122         # the file
1123         mtime = os.stat(template_file).st_mtime
1124
1125         # invoke the editor
1126         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1127             editor = os.environ.get("P4EDITOR")
1128         else:
1129             editor = read_pipe("git var GIT_EDITOR").strip()
1130         system(editor + " " + template_file)
1131
1132         # If the file was not saved, prompt to see if this patch should
1133         # be skipped.  But skip this verification step if configured so.
1134         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
1135             return True
1136
1137         # modification time updated means user saved the file
1138         if os.stat(template_file).st_mtime > mtime:
1139             return True
1140
1141         while True:
1142             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1143             if response == 'y':
1144                 return True
1145             if response == 'n':
1146                 return False
1147
1148     def applyCommit(self, id):
1149         """Apply one commit, return True if it succeeded."""
1150
1151         print "Applying", read_pipe(["git", "show", "-s",
1152                                      "--format=format:%h %s", id])
1153
1154         (p4User, gitEmail) = self.p4UserForCommit(id)
1155
1156         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1157         filesToAdd = set()
1158         filesToDelete = set()
1159         editedFiles = set()
1160         pureRenameCopy = set()
1161         filesToChangeExecBit = {}
1162
1163         for line in diff:
1164             diff = parseDiffTreeEntry(line)
1165             modifier = diff['status']
1166             path = diff['src']
1167             if modifier == "M":
1168                 p4_edit(path)
1169                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1170                     filesToChangeExecBit[path] = diff['dst_mode']
1171                 editedFiles.add(path)
1172             elif modifier == "A":
1173                 filesToAdd.add(path)
1174                 filesToChangeExecBit[path] = diff['dst_mode']
1175                 if path in filesToDelete:
1176                     filesToDelete.remove(path)
1177             elif modifier == "D":
1178                 filesToDelete.add(path)
1179                 if path in filesToAdd:
1180                     filesToAdd.remove(path)
1181             elif modifier == "C":
1182                 src, dest = diff['src'], diff['dst']
1183                 p4_integrate(src, dest)
1184                 pureRenameCopy.add(dest)
1185                 if diff['src_sha1'] != diff['dst_sha1']:
1186                     p4_edit(dest)
1187                     pureRenameCopy.discard(dest)
1188                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1189                     p4_edit(dest)
1190                     pureRenameCopy.discard(dest)
1191                     filesToChangeExecBit[dest] = diff['dst_mode']
1192                 os.unlink(dest)
1193                 editedFiles.add(dest)
1194             elif modifier == "R":
1195                 src, dest = diff['src'], diff['dst']
1196                 if self.p4HasMoveCommand:
1197                     p4_edit(src)        # src must be open before move
1198                     p4_move(src, dest)  # opens for (move/delete, move/add)
1199                 else:
1200                     p4_integrate(src, dest)
1201                     if diff['src_sha1'] != diff['dst_sha1']:
1202                         p4_edit(dest)
1203                     else:
1204                         pureRenameCopy.add(dest)
1205                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1206                     if not self.p4HasMoveCommand:
1207                         p4_edit(dest)   # with move: already open, writable
1208                     filesToChangeExecBit[dest] = diff['dst_mode']
1209                 if not self.p4HasMoveCommand:
1210                     os.unlink(dest)
1211                     filesToDelete.add(src)
1212                 editedFiles.add(dest)
1213             else:
1214                 die("unknown modifier %s for %s" % (modifier, path))
1215
1216         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1217         patchcmd = diffcmd + " | git apply "
1218         tryPatchCmd = patchcmd + "--check -"
1219         applyPatchCmd = patchcmd + "--check --apply -"
1220         patch_succeeded = True
1221
1222         if os.system(tryPatchCmd) != 0:
1223             fixed_rcs_keywords = False
1224             patch_succeeded = False
1225             print "Unfortunately applying the change failed!"
1226
1227             # Patch failed, maybe it's just RCS keyword woes. Look through
1228             # the patch to see if that's possible.
1229             if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1230                 file = None
1231                 pattern = None
1232                 kwfiles = {}
1233                 for file in editedFiles | filesToDelete:
1234                     # did this file's delta contain RCS keywords?
1235                     pattern = p4_keywords_regexp_for_file(file)
1236
1237                     if pattern:
1238                         # this file is a possibility...look for RCS keywords.
1239                         regexp = re.compile(pattern, re.VERBOSE)
1240                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1241                             if regexp.search(line):
1242                                 if verbose:
1243                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1244                                 kwfiles[file] = pattern
1245                                 break
1246
1247                 for file in kwfiles:
1248                     if verbose:
1249                         print "zapping %s with %s" % (line,pattern)
1250                     self.patchRCSKeywords(file, kwfiles[file])
1251                     fixed_rcs_keywords = True
1252
1253             if fixed_rcs_keywords:
1254                 print "Retrying the patch with RCS keywords cleaned up"
1255                 if os.system(tryPatchCmd) == 0:
1256                     patch_succeeded = True
1257
1258         if not patch_succeeded:
1259             for f in editedFiles:
1260                 p4_revert(f)
1261             return False
1262
1263         #
1264         # Apply the patch for real, and do add/delete/+x handling.
1265         #
1266         system(applyPatchCmd)
1267
1268         for f in filesToAdd:
1269             p4_add(f)
1270         for f in filesToDelete:
1271             p4_revert(f)
1272             p4_delete(f)
1273
1274         # Set/clear executable bits
1275         for f in filesToChangeExecBit.keys():
1276             mode = filesToChangeExecBit[f]
1277             setP4ExecBit(f, mode)
1278
1279         #
1280         # Build p4 change description, starting with the contents
1281         # of the git commit message.
1282         #
1283         logMessage = extractLogMessageFromGitCommit(id)
1284         logMessage = logMessage.strip()
1285         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1286
1287         template = self.prepareSubmitTemplate()
1288         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1289
1290         if self.preserveUser:
1291            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1292
1293         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1294             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1295             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1296             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1297
1298         separatorLine = "######## everything below this line is just the diff #######\n"
1299
1300         # diff
1301         if os.environ.has_key("P4DIFF"):
1302             del(os.environ["P4DIFF"])
1303         diff = ""
1304         for editedFile in editedFiles:
1305             diff += p4_read_pipe(['diff', '-du',
1306                                   wildcard_encode(editedFile)])
1307
1308         # new file diff
1309         newdiff = ""
1310         for newFile in filesToAdd:
1311             newdiff += "==== new file ====\n"
1312             newdiff += "--- /dev/null\n"
1313             newdiff += "+++ %s\n" % newFile
1314             f = open(newFile, "r")
1315             for line in f.readlines():
1316                 newdiff += "+" + line
1317             f.close()
1318
1319         # change description file: submitTemplate, separatorLine, diff, newdiff
1320         (handle, fileName) = tempfile.mkstemp()
1321         tmpFile = os.fdopen(handle, "w+")
1322         if self.isWindows:
1323             submitTemplate = submitTemplate.replace("\n", "\r\n")
1324             separatorLine = separatorLine.replace("\n", "\r\n")
1325             newdiff = newdiff.replace("\n", "\r\n")
1326         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1327         tmpFile.close()
1328
1329         if self.prepare_p4_only:
1330             #
1331             # Leave the p4 tree prepared, and the submit template around
1332             # and let the user decide what to do next
1333             #
1334             print
1335             print "P4 workspace prepared for submission."
1336             print "To submit or revert, go to client workspace"
1337             print "  " + self.clientPath
1338             print
1339             print "To submit, use \"p4 submit\" to write a new description,"
1340             print "or \"p4 submit -i %s\" to use the one prepared by" \
1341                   " \"git p4\"." % fileName
1342             print "You can delete the file \"%s\" when finished." % fileName
1343
1344             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1345                 print "To preserve change ownership by user %s, you must\n" \
1346                       "do \"p4 change -f <change>\" after submitting and\n" \
1347                       "edit the User field."
1348             if pureRenameCopy:
1349                 print "After submitting, renamed files must be re-synced."
1350                 print "Invoke \"p4 sync -f\" on each of these files:"
1351                 for f in pureRenameCopy:
1352                     print "  " + f
1353
1354             print
1355             print "To revert the changes, use \"p4 revert ...\", and delete"
1356             print "the submit template file \"%s\"" % fileName
1357             if filesToAdd:
1358                 print "Since the commit adds new files, they must be deleted:"
1359                 for f in filesToAdd:
1360                     print "  " + f
1361             print
1362             return True
1363
1364         #
1365         # Let the user edit the change description, then submit it.
1366         #
1367         if self.edit_template(fileName):
1368             # read the edited message and submit
1369             ret = True
1370             tmpFile = open(fileName, "rb")
1371             message = tmpFile.read()
1372             tmpFile.close()
1373             submitTemplate = message[:message.index(separatorLine)]
1374             if self.isWindows:
1375                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1376             p4_write_pipe(['submit', '-i'], submitTemplate)
1377
1378             if self.preserveUser:
1379                 if p4User:
1380                     # Get last changelist number. Cannot easily get it from
1381                     # the submit command output as the output is
1382                     # unmarshalled.
1383                     changelist = self.lastP4Changelist()
1384                     self.modifyChangelistUser(changelist, p4User)
1385
1386             # The rename/copy happened by applying a patch that created a
1387             # new file.  This leaves it writable, which confuses p4.
1388             for f in pureRenameCopy:
1389                 p4_sync(f, "-f")
1390
1391         else:
1392             # skip this patch
1393             ret = False
1394             print "Submission cancelled, undoing p4 changes."
1395             for f in editedFiles:
1396                 p4_revert(f)
1397             for f in filesToAdd:
1398                 p4_revert(f)
1399                 os.remove(f)
1400             for f in filesToDelete:
1401                 p4_revert(f)
1402
1403         os.remove(fileName)
1404         return ret
1405
1406     # Export git tags as p4 labels. Create a p4 label and then tag
1407     # with that.
1408     def exportGitTags(self, gitTags):
1409         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1410         if len(validLabelRegexp) == 0:
1411             validLabelRegexp = defaultLabelRegexp
1412         m = re.compile(validLabelRegexp)
1413
1414         for name in gitTags:
1415
1416             if not m.match(name):
1417                 if verbose:
1418                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1419                 continue
1420
1421             # Get the p4 commit this corresponds to
1422             logMessage = extractLogMessageFromGitCommit(name)
1423             values = extractSettingsGitLog(logMessage)
1424
1425             if not values.has_key('change'):
1426                 # a tag pointing to something not sent to p4; ignore
1427                 if verbose:
1428                     print "git tag %s does not give a p4 commit" % name
1429                 continue
1430             else:
1431                 changelist = values['change']
1432
1433             # Get the tag details.
1434             inHeader = True
1435             isAnnotated = False
1436             body = []
1437             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1438                 l = l.strip()
1439                 if inHeader:
1440                     if re.match(r'tag\s+', l):
1441                         isAnnotated = True
1442                     elif re.match(r'\s*$', l):
1443                         inHeader = False
1444                         continue
1445                 else:
1446                     body.append(l)
1447
1448             if not isAnnotated:
1449                 body = ["lightweight tag imported by git p4\n"]
1450
1451             # Create the label - use the same view as the client spec we are using
1452             clientSpec = getClientSpec()
1453
1454             labelTemplate  = "Label: %s\n" % name
1455             labelTemplate += "Description:\n"
1456             for b in body:
1457                 labelTemplate += "\t" + b + "\n"
1458             labelTemplate += "View:\n"
1459             for mapping in clientSpec.mappings:
1460                 labelTemplate += "\t%s\n" % mapping.depot_side.path
1461
1462             if self.dry_run:
1463                 print "Would create p4 label %s for tag" % name
1464             elif self.prepare_p4_only:
1465                 print "Not creating p4 label %s for tag due to option" \
1466                       " --prepare-p4-only" % name
1467             else:
1468                 p4_write_pipe(["label", "-i"], labelTemplate)
1469
1470                 # Use the label
1471                 p4_system(["tag", "-l", name] +
1472                           ["%s@%s" % (mapping.depot_side.path, changelist) for mapping in clientSpec.mappings])
1473
1474                 if verbose:
1475                     print "created p4 label for tag %s" % name
1476
1477     def run(self, args):
1478         if len(args) == 0:
1479             self.master = currentGitBranch()
1480             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1481                 die("Detecting current git branch failed!")
1482         elif len(args) == 1:
1483             self.master = args[0]
1484             if not branchExists(self.master):
1485                 die("Branch %s does not exist" % self.master)
1486         else:
1487             return False
1488
1489         allowSubmit = gitConfig("git-p4.allowSubmit")
1490         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1491             die("%s is not in git-p4.allowSubmit" % self.master)
1492
1493         [upstream, settings] = findUpstreamBranchPoint()
1494         self.depotPath = settings['depot-paths'][0]
1495         if len(self.origin) == 0:
1496             self.origin = upstream
1497
1498         if self.preserveUser:
1499             if not self.canChangeChangelists():
1500                 die("Cannot preserve user names without p4 super-user or admin permissions")
1501
1502         # if not set from the command line, try the config file
1503         if self.conflict_behavior is None:
1504             val = gitConfig("git-p4.conflict")
1505             if val:
1506                 if val not in self.conflict_behavior_choices:
1507                     die("Invalid value '%s' for config git-p4.conflict" % val)
1508             else:
1509                 val = "ask"
1510             self.conflict_behavior = val
1511
1512         if self.verbose:
1513             print "Origin branch is " + self.origin
1514
1515         if len(self.depotPath) == 0:
1516             print "Internal error: cannot locate perforce depot path from existing branches"
1517             sys.exit(128)
1518
1519         self.useClientSpec = False
1520         if gitConfig("git-p4.useclientspec", "--bool") == "true":
1521             self.useClientSpec = True
1522         if self.useClientSpec:
1523             self.clientSpecDirs = getClientSpec()
1524
1525         if self.useClientSpec:
1526             # all files are relative to the client spec
1527             self.clientPath = getClientRoot()
1528         else:
1529             self.clientPath = p4Where(self.depotPath)
1530
1531         if self.clientPath == "":
1532             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1533
1534         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1535         self.oldWorkingDirectory = os.getcwd()
1536
1537         # ensure the clientPath exists
1538         new_client_dir = False
1539         if not os.path.exists(self.clientPath):
1540             new_client_dir = True
1541             os.makedirs(self.clientPath)
1542
1543         chdir(self.clientPath)
1544         if self.dry_run:
1545             print "Would synchronize p4 checkout in %s" % self.clientPath
1546         else:
1547             print "Synchronizing p4 checkout..."
1548             if new_client_dir:
1549                 # old one was destroyed, and maybe nobody told p4
1550                 p4_sync("...", "-f")
1551             else:
1552                 p4_sync("...")
1553         self.check()
1554
1555         commits = []
1556         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1557             commits.append(line.strip())
1558         commits.reverse()
1559
1560         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1561             self.checkAuthorship = False
1562         else:
1563             self.checkAuthorship = True
1564
1565         if self.preserveUser:
1566             self.checkValidP4Users(commits)
1567
1568         #
1569         # Build up a set of options to be passed to diff when
1570         # submitting each commit to p4.
1571         #
1572         if self.detectRenames:
1573             # command-line -M arg
1574             self.diffOpts = "-M"
1575         else:
1576             # If not explicitly set check the config variable
1577             detectRenames = gitConfig("git-p4.detectRenames")
1578
1579             if detectRenames.lower() == "false" or detectRenames == "":
1580                 self.diffOpts = ""
1581             elif detectRenames.lower() == "true":
1582                 self.diffOpts = "-M"
1583             else:
1584                 self.diffOpts = "-M%s" % detectRenames
1585
1586         # no command-line arg for -C or --find-copies-harder, just
1587         # config variables
1588         detectCopies = gitConfig("git-p4.detectCopies")
1589         if detectCopies.lower() == "false" or detectCopies == "":
1590             pass
1591         elif detectCopies.lower() == "true":
1592             self.diffOpts += " -C"
1593         else:
1594             self.diffOpts += " -C%s" % detectCopies
1595
1596         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
1597             self.diffOpts += " --find-copies-harder"
1598
1599         #
1600         # Apply the commits, one at a time.  On failure, ask if should
1601         # continue to try the rest of the patches, or quit.
1602         #
1603         if self.dry_run:
1604             print "Would apply"
1605         applied = []
1606         last = len(commits) - 1
1607         for i, commit in enumerate(commits):
1608             if self.dry_run:
1609                 print " ", read_pipe(["git", "show", "-s",
1610                                       "--format=format:%h %s", commit])
1611                 ok = True
1612             else:
1613                 ok = self.applyCommit(commit)
1614             if ok:
1615                 applied.append(commit)
1616             else:
1617                 if self.prepare_p4_only and i < last:
1618                     print "Processing only the first commit due to option" \
1619                           " --prepare-p4-only"
1620                     break
1621                 if i < last:
1622                     quit = False
1623                     while True:
1624                         # prompt for what to do, or use the option/variable
1625                         if self.conflict_behavior == "ask":
1626                             print "What do you want to do?"
1627                             response = raw_input("[s]kip this commit but apply"
1628                                                  " the rest, or [q]uit? ")
1629                             if not response:
1630                                 continue
1631                         elif self.conflict_behavior == "skip":
1632                             response = "s"
1633                         elif self.conflict_behavior == "quit":
1634                             response = "q"
1635                         else:
1636                             die("Unknown conflict_behavior '%s'" %
1637                                 self.conflict_behavior)
1638
1639                         if response[0] == "s":
1640                             print "Skipping this commit, but applying the rest"
1641                             break
1642                         if response[0] == "q":
1643                             print "Quitting"
1644                             quit = True
1645                             break
1646                     if quit:
1647                         break
1648
1649         chdir(self.oldWorkingDirectory)
1650
1651         if self.dry_run:
1652             pass
1653         elif self.prepare_p4_only:
1654             pass
1655         elif len(commits) == len(applied):
1656             print "All commits applied!"
1657
1658             sync = P4Sync()
1659             sync.run([])
1660
1661             rebase = P4Rebase()
1662             rebase.rebase()
1663
1664         else:
1665             if len(applied) == 0:
1666                 print "No commits applied."
1667             else:
1668                 print "Applied only the commits marked with '*':"
1669                 for c in commits:
1670                     if c in applied:
1671                         star = "*"
1672                     else:
1673                         star = " "
1674                     print star, read_pipe(["git", "show", "-s",
1675                                            "--format=format:%h %s",  c])
1676                 print "You will have to do 'git p4 sync' and rebase."
1677
1678         if gitConfig("git-p4.exportLabels", "--bool") == "true":
1679             self.exportLabels = True
1680
1681         if self.exportLabels:
1682             p4Labels = getP4Labels(self.depotPath)
1683             gitTags = getGitTags()
1684
1685             missingGitTags = gitTags - p4Labels
1686             self.exportGitTags(missingGitTags)
1687
1688         # exit with error unless everything applied perfecly
1689         if len(commits) != len(applied):
1690                 sys.exit(1)
1691
1692         return True
1693
1694 class View(object):
1695     """Represent a p4 view ("p4 help views"), and map files in a
1696        repo according to the view."""
1697
1698     class Path(object):
1699         """A depot or client path, possibly containing wildcards.
1700            The only one supported is ... at the end, currently.
1701            Initialize with the full path, with //depot or //client."""
1702
1703         def __init__(self, path, is_depot):
1704             self.path = path
1705             self.is_depot = is_depot
1706             self.find_wildcards()
1707             # remember the prefix bit, useful for relative mappings
1708             m = re.match("(//[^/]+/)", self.path)
1709             if not m:
1710                 die("Path %s does not start with //prefix/" % self.path)
1711             prefix = m.group(1)
1712             if not self.is_depot:
1713                 # strip //client/ on client paths
1714                 self.path = self.path[len(prefix):]
1715
1716         def find_wildcards(self):
1717             """Make sure wildcards are valid, and set up internal
1718                variables."""
1719
1720             self.ends_triple_dot = False
1721             # There are three wildcards allowed in p4 views
1722             # (see "p4 help views").  This code knows how to
1723             # handle "..." (only at the end), but cannot deal with
1724             # "%%n" or "*".  Only check the depot_side, as p4 should
1725             # validate that the client_side matches too.
1726             if re.search(r'%%[1-9]', self.path):
1727                 die("Can't handle %%n wildcards in view: %s" % self.path)
1728             if self.path.find("*") >= 0:
1729                 die("Can't handle * wildcards in view: %s" % self.path)
1730             triple_dot_index = self.path.find("...")
1731             if triple_dot_index >= 0:
1732                 if triple_dot_index != len(self.path) - 3:
1733                     die("Can handle only single ... wildcard, at end: %s" %
1734                         self.path)
1735                 self.ends_triple_dot = True
1736
1737         def ensure_compatible(self, other_path):
1738             """Make sure the wildcards agree."""
1739             if self.ends_triple_dot != other_path.ends_triple_dot:
1740                  die("Both paths must end with ... if either does;\n" +
1741                      "paths: %s %s" % (self.path, other_path.path))
1742
1743         def match_wildcards(self, test_path):
1744             """See if this test_path matches us, and fill in the value
1745                of the wildcards if so.  Returns a tuple of
1746                (True|False, wildcards[]).  For now, only the ... at end
1747                is supported, so at most one wildcard."""
1748             if self.ends_triple_dot:
1749                 dotless = self.path[:-3]
1750                 if test_path.startswith(dotless):
1751                     wildcard = test_path[len(dotless):]
1752                     return (True, [ wildcard ])
1753             else:
1754                 if test_path == self.path:
1755                     return (True, [])
1756             return (False, [])
1757
1758         def match(self, test_path):
1759             """Just return if it matches; don't bother with the wildcards."""
1760             b, _ = self.match_wildcards(test_path)
1761             return b
1762
1763         def fill_in_wildcards(self, wildcards):
1764             """Return the relative path, with the wildcards filled in
1765                if there are any."""
1766             if self.ends_triple_dot:
1767                 return self.path[:-3] + wildcards[0]
1768             else:
1769                 return self.path
1770
1771     class Mapping(object):
1772         def __init__(self, depot_side, client_side, overlay, exclude):
1773             # depot_side is without the trailing /... if it had one
1774             self.depot_side = View.Path(depot_side, is_depot=True)
1775             self.client_side = View.Path(client_side, is_depot=False)
1776             self.overlay = overlay  # started with "+"
1777             self.exclude = exclude  # started with "-"
1778             assert not (self.overlay and self.exclude)
1779             self.depot_side.ensure_compatible(self.client_side)
1780
1781         def __str__(self):
1782             c = " "
1783             if self.overlay:
1784                 c = "+"
1785             if self.exclude:
1786                 c = "-"
1787             return "View.Mapping: %s%s -> %s" % \
1788                    (c, self.depot_side.path, self.client_side.path)
1789
1790         def map_depot_to_client(self, depot_path):
1791             """Calculate the client path if using this mapping on the
1792                given depot path; does not consider the effect of other
1793                mappings in a view.  Even excluded mappings are returned."""
1794             matches, wildcards = self.depot_side.match_wildcards(depot_path)
1795             if not matches:
1796                 return ""
1797             client_path = self.client_side.fill_in_wildcards(wildcards)
1798             return client_path
1799
1800     #
1801     # View methods
1802     #
1803     def __init__(self):
1804         self.mappings = []
1805
1806     def append(self, view_line):
1807         """Parse a view line, splitting it into depot and client
1808            sides.  Append to self.mappings, preserving order."""
1809
1810         # Split the view line into exactly two words.  P4 enforces
1811         # structure on these lines that simplifies this quite a bit.
1812         #
1813         # Either or both words may be double-quoted.
1814         # Single quotes do not matter.
1815         # Double-quote marks cannot occur inside the words.
1816         # A + or - prefix is also inside the quotes.
1817         # There are no quotes unless they contain a space.
1818         # The line is already white-space stripped.
1819         # The two words are separated by a single space.
1820         #
1821         if view_line[0] == '"':
1822             # First word is double quoted.  Find its end.
1823             close_quote_index = view_line.find('"', 1)
1824             if close_quote_index <= 0:
1825                 die("No first-word closing quote found: %s" % view_line)
1826             depot_side = view_line[1:close_quote_index]
1827             # skip closing quote and space
1828             rhs_index = close_quote_index + 1 + 1
1829         else:
1830             space_index = view_line.find(" ")
1831             if space_index <= 0:
1832                 die("No word-splitting space found: %s" % view_line)
1833             depot_side = view_line[0:space_index]
1834             rhs_index = space_index + 1
1835
1836         if view_line[rhs_index] == '"':
1837             # Second word is double quoted.  Make sure there is a
1838             # double quote at the end too.
1839             if not view_line.endswith('"'):
1840                 die("View line with rhs quote should end with one: %s" %
1841                     view_line)
1842             # skip the quotes
1843             client_side = view_line[rhs_index+1:-1]
1844         else:
1845             client_side = view_line[rhs_index:]
1846
1847         # prefix + means overlay on previous mapping
1848         overlay = False
1849         if depot_side.startswith("+"):
1850             overlay = True
1851             depot_side = depot_side[1:]
1852
1853         # prefix - means exclude this path
1854         exclude = False
1855         if depot_side.startswith("-"):
1856             exclude = True
1857             depot_side = depot_side[1:]
1858
1859         m = View.Mapping(depot_side, client_side, overlay, exclude)
1860         self.mappings.append(m)
1861
1862     def map_in_client(self, depot_path):
1863         """Return the relative location in the client where this
1864            depot file should live.  Returns "" if the file should
1865            not be mapped in the client."""
1866
1867         paths_filled = []
1868         client_path = ""
1869
1870         # look at later entries first
1871         for m in self.mappings[::-1]:
1872
1873             # see where will this path end up in the client
1874             p = m.map_depot_to_client(depot_path)
1875
1876             if p == "":
1877                 # Depot path does not belong in client.  Must remember
1878                 # this, as previous items should not cause files to
1879                 # exist in this path either.  Remember that the list is
1880                 # being walked from the end, which has higher precedence.
1881                 # Overlap mappings do not exclude previous mappings.
1882                 if not m.overlay:
1883                     paths_filled.append(m.client_side)
1884
1885             else:
1886                 # This mapping matched; no need to search any further.
1887                 # But, the mapping could be rejected if the client path
1888                 # has already been claimed by an earlier mapping (i.e.
1889                 # one later in the list, which we are walking backwards).
1890                 already_mapped_in_client = False
1891                 for f in paths_filled:
1892                     # this is View.Path.match
1893                     if f.match(p):
1894                         already_mapped_in_client = True
1895                         break
1896                 if not already_mapped_in_client:
1897                     # Include this file, unless it is from a line that
1898                     # explicitly said to exclude it.
1899                     if not m.exclude:
1900                         client_path = p
1901
1902                 # a match, even if rejected, always stops the search
1903                 break
1904
1905         return client_path
1906
1907 class P4Sync(Command, P4UserMap):
1908     delete_actions = ( "delete", "move/delete", "purge" )
1909
1910     def __init__(self):
1911         Command.__init__(self)
1912         P4UserMap.__init__(self)
1913         self.options = [
1914                 optparse.make_option("--branch", dest="branch"),
1915                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1916                 optparse.make_option("--changesfile", dest="changesFile"),
1917                 optparse.make_option("--silent", dest="silent", action="store_true"),
1918                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1919                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1920                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1921                                      help="Import into refs/heads/ , not refs/remotes"),
1922                 optparse.make_option("--max-changes", dest="maxChanges"),
1923                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1924                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1925                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1926                                      help="Only sync files that are included in the Perforce Client Spec")
1927         ]
1928         self.description = """Imports from Perforce into a git repository.\n
1929     example:
1930     //depot/my/project/ -- to import the current head
1931     //depot/my/project/@all -- to import everything
1932     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1933
1934     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1935
1936         self.usage += " //depot/path[@revRange]"
1937         self.silent = False
1938         self.createdBranches = set()
1939         self.committedChanges = set()
1940         self.branch = ""
1941         self.detectBranches = False
1942         self.detectLabels = False
1943         self.importLabels = False
1944         self.changesFile = ""
1945         self.syncWithOrigin = True
1946         self.importIntoRemotes = True
1947         self.maxChanges = ""
1948         self.isWindows = (platform.system() == "Windows")
1949         self.keepRepoPath = False
1950         self.depotPaths = None
1951         self.p4BranchesInGit = []
1952         self.cloneExclude = []
1953         self.useClientSpec = False
1954         self.useClientSpec_from_options = False
1955         self.clientSpecDirs = None
1956         self.tempBranches = []
1957         self.tempBranchLocation = "git-p4-tmp"
1958
1959         if gitConfig("git-p4.syncFromOrigin") == "false":
1960             self.syncWithOrigin = False
1961
1962     # Force a checkpoint in fast-import and wait for it to finish
1963     def checkpoint(self):
1964         self.gitStream.write("checkpoint\n\n")
1965         self.gitStream.write("progress checkpoint\n\n")
1966         out = self.gitOutput.readline()
1967         if self.verbose:
1968             print "checkpoint finished: " + out
1969
1970     def extractFilesFromCommit(self, commit):
1971         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1972                              for path in self.cloneExclude]
1973         files = []
1974         fnum = 0
1975         while commit.has_key("depotFile%s" % fnum):
1976             path =  commit["depotFile%s" % fnum]
1977
1978             if [p for p in self.cloneExclude
1979                 if p4PathStartsWith(path, p)]:
1980                 found = False
1981             else:
1982                 found = [p for p in self.depotPaths
1983                          if p4PathStartsWith(path, p)]
1984             if not found:
1985                 fnum = fnum + 1
1986                 continue
1987
1988             file = {}
1989             file["path"] = path
1990             file["rev"] = commit["rev%s" % fnum]
1991             file["action"] = commit["action%s" % fnum]
1992             file["type"] = commit["type%s" % fnum]
1993             files.append(file)
1994             fnum = fnum + 1
1995         return files
1996
1997     def stripRepoPath(self, path, prefixes):
1998         """When streaming files, this is called to map a p4 depot path
1999            to where it should go in git.  The prefixes are either
2000            self.depotPaths, or self.branchPrefixes in the case of
2001            branch detection."""
2002
2003         if self.useClientSpec:
2004             # branch detection moves files up a level (the branch name)
2005             # from what client spec interpretation gives
2006             path = self.clientSpecDirs.map_in_client(path)
2007             if self.detectBranches:
2008                 for b in self.knownBranches:
2009                     if path.startswith(b + "/"):
2010                         path = path[len(b)+1:]
2011
2012         elif self.keepRepoPath:
2013             # Preserve everything in relative path name except leading
2014             # //depot/; just look at first prefix as they all should
2015             # be in the same depot.
2016             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2017             if p4PathStartsWith(path, depot):
2018                 path = path[len(depot):]
2019
2020         else:
2021             for p in prefixes:
2022                 if p4PathStartsWith(path, p):
2023                     path = path[len(p):]
2024                     break
2025
2026         path = wildcard_decode(path)
2027         return path
2028
2029     def splitFilesIntoBranches(self, commit):
2030         """Look at each depotFile in the commit to figure out to what
2031            branch it belongs."""
2032
2033         branches = {}
2034         fnum = 0
2035         while commit.has_key("depotFile%s" % fnum):
2036             path =  commit["depotFile%s" % fnum]
2037             found = [p for p in self.depotPaths
2038                      if p4PathStartsWith(path, p)]
2039             if not found:
2040                 fnum = fnum + 1
2041                 continue
2042
2043             file = {}
2044             file["path"] = path
2045             file["rev"] = commit["rev%s" % fnum]
2046             file["action"] = commit["action%s" % fnum]
2047             file["type"] = commit["type%s" % fnum]
2048             fnum = fnum + 1
2049
2050             # start with the full relative path where this file would
2051             # go in a p4 client
2052             if self.useClientSpec:
2053                 relPath = self.clientSpecDirs.map_in_client(path)
2054             else:
2055                 relPath = self.stripRepoPath(path, self.depotPaths)
2056
2057             for branch in self.knownBranches.keys():
2058                 # add a trailing slash so that a commit into qt/4.2foo
2059                 # doesn't end up in qt/4.2, e.g.
2060                 if relPath.startswith(branch + "/"):
2061                     if branch not in branches:
2062                         branches[branch] = []
2063                     branches[branch].append(file)
2064                     break
2065
2066         return branches
2067
2068     # output one file from the P4 stream
2069     # - helper for streamP4Files
2070
2071     def streamOneP4File(self, file, contents):
2072         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2073         if verbose:
2074             sys.stderr.write("%s\n" % relPath)
2075
2076         (type_base, type_mods) = split_p4_type(file["type"])
2077
2078         git_mode = "100644"
2079         if "x" in type_mods:
2080             git_mode = "100755"
2081         if type_base == "symlink":
2082             git_mode = "120000"
2083             # p4 print on a symlink contains "target\n"; remove the newline
2084             data = ''.join(contents)
2085             contents = [data[:-1]]
2086
2087         if type_base == "utf16":
2088             # p4 delivers different text in the python output to -G
2089             # than it does when using "print -o", or normal p4 client
2090             # operations.  utf16 is converted to ascii or utf8, perhaps.
2091             # But ascii text saved as -t utf16 is completely mangled.
2092             # Invoke print -o to get the real contents.
2093             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2094             contents = [ text ]
2095
2096         if type_base == "apple":
2097             # Apple filetype files will be streamed as a concatenation of
2098             # its appledouble header and the contents.  This is useless
2099             # on both macs and non-macs.  If using "print -q -o xx", it
2100             # will create "xx" with the data, and "%xx" with the header.
2101             # This is also not very useful.
2102             #
2103             # Ideally, someday, this script can learn how to generate
2104             # appledouble files directly and import those to git, but
2105             # non-mac machines can never find a use for apple filetype.
2106             print "\nIgnoring apple filetype file %s" % file['depotFile']
2107             return
2108
2109         # Perhaps windows wants unicode, utf16 newlines translated too;
2110         # but this is not doing it.
2111         if self.isWindows and type_base == "text":
2112             mangled = []
2113             for data in contents:
2114                 data = data.replace("\r\n", "\n")
2115                 mangled.append(data)
2116             contents = mangled
2117
2118         # Note that we do not try to de-mangle keywords on utf16 files,
2119         # even though in theory somebody may want that.
2120         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2121         if pattern:
2122             regexp = re.compile(pattern, re.VERBOSE)
2123             text = ''.join(contents)
2124             text = regexp.sub(r'$\1$', text)
2125             contents = [ text ]
2126
2127         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2128
2129         # total length...
2130         length = 0
2131         for d in contents:
2132             length = length + len(d)
2133
2134         self.gitStream.write("data %d\n" % length)
2135         for d in contents:
2136             self.gitStream.write(d)
2137         self.gitStream.write("\n")
2138
2139     def streamOneP4Deletion(self, file):
2140         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2141         if verbose:
2142             sys.stderr.write("delete %s\n" % relPath)
2143         self.gitStream.write("D %s\n" % relPath)
2144
2145     # handle another chunk of streaming data
2146     def streamP4FilesCb(self, marshalled):
2147
2148         # catch p4 errors and complain
2149         err = None
2150         if "code" in marshalled:
2151             if marshalled["code"] == "error":
2152                 if "data" in marshalled:
2153                     err = marshalled["data"].rstrip()
2154         if err:
2155             f = None
2156             if self.stream_have_file_info:
2157                 if "depotFile" in self.stream_file:
2158                     f = self.stream_file["depotFile"]
2159             # force a failure in fast-import, else an empty
2160             # commit will be made
2161             self.gitStream.write("\n")
2162             self.gitStream.write("die-now\n")
2163             self.gitStream.close()
2164             # ignore errors, but make sure it exits first
2165             self.importProcess.wait()
2166             if f:
2167                 die("Error from p4 print for %s: %s" % (f, err))
2168             else:
2169                 die("Error from p4 print: %s" % err)
2170
2171         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2172             # start of a new file - output the old one first
2173             self.streamOneP4File(self.stream_file, self.stream_contents)
2174             self.stream_file = {}
2175             self.stream_contents = []
2176             self.stream_have_file_info = False
2177
2178         # pick up the new file information... for the
2179         # 'data' field we need to append to our array
2180         for k in marshalled.keys():
2181             if k == 'data':
2182                 self.stream_contents.append(marshalled['data'])
2183             else:
2184                 self.stream_file[k] = marshalled[k]
2185
2186         self.stream_have_file_info = True
2187
2188     # Stream directly from "p4 files" into "git fast-import"
2189     def streamP4Files(self, files):
2190         filesForCommit = []
2191         filesToRead = []
2192         filesToDelete = []
2193
2194         for f in files:
2195             # if using a client spec, only add the files that have
2196             # a path in the client
2197             if self.clientSpecDirs:
2198                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2199                     continue
2200
2201             filesForCommit.append(f)
2202             if f['action'] in self.delete_actions:
2203                 filesToDelete.append(f)
2204             else:
2205                 filesToRead.append(f)
2206
2207         # deleted files...
2208         for f in filesToDelete:
2209             self.streamOneP4Deletion(f)
2210
2211         if len(filesToRead) > 0:
2212             self.stream_file = {}
2213             self.stream_contents = []
2214             self.stream_have_file_info = False
2215
2216             # curry self argument
2217             def streamP4FilesCbSelf(entry):
2218                 self.streamP4FilesCb(entry)
2219
2220             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2221
2222             p4CmdList(["-x", "-", "print"],
2223                       stdin=fileArgs,
2224                       cb=streamP4FilesCbSelf)
2225
2226             # do the last chunk
2227             if self.stream_file.has_key('depotFile'):
2228                 self.streamOneP4File(self.stream_file, self.stream_contents)
2229
2230     def make_email(self, userid):
2231         if userid in self.users:
2232             return self.users[userid]
2233         else:
2234             return "%s <a@b>" % userid
2235
2236     # Stream a p4 tag
2237     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2238         if verbose:
2239             print "writing tag %s for commit %s" % (labelName, commit)
2240         gitStream.write("tag %s\n" % labelName)
2241         gitStream.write("from %s\n" % commit)
2242
2243         if labelDetails.has_key('Owner'):
2244             owner = labelDetails["Owner"]
2245         else:
2246             owner = None
2247
2248         # Try to use the owner of the p4 label, or failing that,
2249         # the current p4 user id.
2250         if owner:
2251             email = self.make_email(owner)
2252         else:
2253             email = self.make_email(self.p4UserId())
2254         tagger = "%s %s %s" % (email, epoch, self.tz)
2255
2256         gitStream.write("tagger %s\n" % tagger)
2257
2258         print "labelDetails=",labelDetails
2259         if labelDetails.has_key('Description'):
2260             description = labelDetails['Description']
2261         else:
2262             description = 'Label from git p4'
2263
2264         gitStream.write("data %d\n" % len(description))
2265         gitStream.write(description)
2266         gitStream.write("\n")
2267
2268     def commit(self, details, files, branch, parent = ""):
2269         epoch = details["time"]
2270         author = details["user"]
2271
2272         if self.verbose:
2273             print "commit into %s" % branch
2274
2275         # start with reading files; if that fails, we should not
2276         # create a commit.
2277         new_files = []
2278         for f in files:
2279             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2280                 new_files.append (f)
2281             else:
2282                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2283
2284         self.gitStream.write("commit %s\n" % branch)
2285 #        gitStream.write("mark :%s\n" % details["change"])
2286         self.committedChanges.add(int(details["change"]))
2287         committer = ""
2288         if author not in self.users:
2289             self.getUserMapFromPerforceServer()
2290         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2291
2292         self.gitStream.write("committer %s\n" % committer)
2293
2294         self.gitStream.write("data <<EOT\n")
2295         self.gitStream.write(details["desc"])
2296         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2297                              (','.join(self.branchPrefixes), details["change"]))
2298         if len(details['options']) > 0:
2299             self.gitStream.write(": options = %s" % details['options'])
2300         self.gitStream.write("]\nEOT\n\n")
2301
2302         if len(parent) > 0:
2303             if self.verbose:
2304                 print "parent %s" % parent
2305             self.gitStream.write("from %s\n" % parent)
2306
2307         self.streamP4Files(new_files)
2308         self.gitStream.write("\n")
2309
2310         change = int(details["change"])
2311
2312         if self.labels.has_key(change):
2313             label = self.labels[change]
2314             labelDetails = label[0]
2315             labelRevisions = label[1]
2316             if self.verbose:
2317                 print "Change %s is labelled %s" % (change, labelDetails)
2318
2319             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2320                                                 for p in self.branchPrefixes])
2321
2322             if len(files) == len(labelRevisions):
2323
2324                 cleanedFiles = {}
2325                 for info in files:
2326                     if info["action"] in self.delete_actions:
2327                         continue
2328                     cleanedFiles[info["depotFile"]] = info["rev"]
2329
2330                 if cleanedFiles == labelRevisions:
2331                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2332
2333                 else:
2334                     if not self.silent:
2335                         print ("Tag %s does not match with change %s: files do not match."
2336                                % (labelDetails["label"], change))
2337
2338             else:
2339                 if not self.silent:
2340                     print ("Tag %s does not match with change %s: file count is different."
2341                            % (labelDetails["label"], change))
2342
2343     # Build a dictionary of changelists and labels, for "detect-labels" option.
2344     def getLabels(self):
2345         self.labels = {}
2346
2347         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2348         if len(l) > 0 and not self.silent:
2349             print "Finding files belonging to labels in %s" % `self.depotPaths`
2350
2351         for output in l:
2352             label = output["label"]
2353             revisions = {}
2354             newestChange = 0
2355             if self.verbose:
2356                 print "Querying files for label %s" % label
2357             for file in p4CmdList(["files"] +
2358                                       ["%s...@%s" % (p, label)
2359                                           for p in self.depotPaths]):
2360                 revisions[file["depotFile"]] = file["rev"]
2361                 change = int(file["change"])
2362                 if change > newestChange:
2363                     newestChange = change
2364
2365             self.labels[newestChange] = [output, revisions]
2366
2367         if self.verbose:
2368             print "Label changes: %s" % self.labels.keys()
2369
2370     # Import p4 labels as git tags. A direct mapping does not
2371     # exist, so assume that if all the files are at the same revision
2372     # then we can use that, or it's something more complicated we should
2373     # just ignore.
2374     def importP4Labels(self, stream, p4Labels):
2375         if verbose:
2376             print "import p4 labels: " + ' '.join(p4Labels)
2377
2378         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2379         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2380         if len(validLabelRegexp) == 0:
2381             validLabelRegexp = defaultLabelRegexp
2382         m = re.compile(validLabelRegexp)
2383
2384         for name in p4Labels:
2385             commitFound = False
2386
2387             if not m.match(name):
2388                 if verbose:
2389                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2390                 continue
2391
2392             if name in ignoredP4Labels:
2393                 continue
2394
2395             labelDetails = p4CmdList(['label', "-o", name])[0]
2396
2397             # get the most recent changelist for each file in this label
2398             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2399                                 for p in self.depotPaths])
2400
2401             if change.has_key('change'):
2402                 # find the corresponding git commit; take the oldest commit
2403                 changelist = int(change['change'])
2404                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2405                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2406                 if len(gitCommit) == 0:
2407                     print "could not find git commit for changelist %d" % changelist
2408                 else:
2409                     gitCommit = gitCommit.strip()
2410                     commitFound = True
2411                     # Convert from p4 time format
2412                     try:
2413                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2414                     except ValueError:
2415                         print "Could not convert label time %s" % labelDetails['Update']
2416                         tmwhen = 1
2417
2418                     when = int(time.mktime(tmwhen))
2419                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2420                     if verbose:
2421                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2422             else:
2423                 if verbose:
2424                     print "Label %s has no changelists - possibly deleted?" % name
2425
2426             if not commitFound:
2427                 # We can't import this label; don't try again as it will get very
2428                 # expensive repeatedly fetching all the files for labels that will
2429                 # never be imported. If the label is moved in the future, the
2430                 # ignore will need to be removed manually.
2431                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2432
2433     def guessProjectName(self):
2434         for p in self.depotPaths:
2435             if p.endswith("/"):
2436                 p = p[:-1]
2437             p = p[p.strip().rfind("/") + 1:]
2438             if not p.endswith("/"):
2439                p += "/"
2440             return p
2441
2442     def getBranchMapping(self):
2443         lostAndFoundBranches = set()
2444
2445         user = gitConfig("git-p4.branchUser")
2446         if len(user) > 0:
2447             command = "branches -u %s" % user
2448         else:
2449             command = "branches"
2450
2451         for info in p4CmdList(command):
2452             details = p4Cmd(["branch", "-o", info["branch"]])
2453             viewIdx = 0
2454             while details.has_key("View%s" % viewIdx):
2455                 paths = details["View%s" % viewIdx].split(" ")
2456                 viewIdx = viewIdx + 1
2457                 # require standard //depot/foo/... //depot/bar/... mapping
2458                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2459                     continue
2460                 source = paths[0]
2461                 destination = paths[1]
2462                 ## HACK
2463                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2464                     source = source[len(self.depotPaths[0]):-4]
2465                     destination = destination[len(self.depotPaths[0]):-4]
2466
2467                     if destination in self.knownBranches:
2468                         if not self.silent:
2469                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2470                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2471                         continue
2472
2473                     self.knownBranches[destination] = source
2474
2475                     lostAndFoundBranches.discard(destination)
2476
2477                     if source not in self.knownBranches:
2478                         lostAndFoundBranches.add(source)
2479
2480         # Perforce does not strictly require branches to be defined, so we also
2481         # check git config for a branch list.
2482         #
2483         # Example of branch definition in git config file:
2484         # [git-p4]
2485         #   branchList=main:branchA
2486         #   branchList=main:branchB
2487         #   branchList=branchA:branchC
2488         configBranches = gitConfigList("git-p4.branchList")
2489         for branch in configBranches:
2490             if branch:
2491                 (source, destination) = branch.split(":")
2492                 self.knownBranches[destination] = source
2493
2494                 lostAndFoundBranches.discard(destination)
2495
2496                 if source not in self.knownBranches:
2497                     lostAndFoundBranches.add(source)
2498
2499
2500         for branch in lostAndFoundBranches:
2501             self.knownBranches[branch] = branch
2502
2503     def getBranchMappingFromGitBranches(self):
2504         branches = p4BranchesInGit(self.importIntoRemotes)
2505         for branch in branches.keys():
2506             if branch == "master":
2507                 branch = "main"
2508             else:
2509                 branch = branch[len(self.projectName):]
2510             self.knownBranches[branch] = branch
2511
2512     def listExistingP4GitBranches(self):
2513         # branches holds mapping from name to commit
2514         branches = p4BranchesInGit(self.importIntoRemotes)
2515         self.p4BranchesInGit = branches.keys()
2516         for branch in branches.keys():
2517             self.initialParents[self.refPrefix + branch] = branches[branch]
2518
2519     def updateOptionDict(self, d):
2520         option_keys = {}
2521         if self.keepRepoPath:
2522             option_keys['keepRepoPath'] = 1
2523
2524         d["options"] = ' '.join(sorted(option_keys.keys()))
2525
2526     def readOptions(self, d):
2527         self.keepRepoPath = (d.has_key('options')
2528                              and ('keepRepoPath' in d['options']))
2529
2530     def gitRefForBranch(self, branch):
2531         if branch == "main":
2532             return self.refPrefix + "master"
2533
2534         if len(branch) <= 0:
2535             return branch
2536
2537         return self.refPrefix + self.projectName + branch
2538
2539     def gitCommitByP4Change(self, ref, change):
2540         if self.verbose:
2541             print "looking in ref " + ref + " for change %s using bisect..." % change
2542
2543         earliestCommit = ""
2544         latestCommit = parseRevision(ref)
2545
2546         while True:
2547             if self.verbose:
2548                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2549             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2550             if len(next) == 0:
2551                 if self.verbose:
2552                     print "argh"
2553                 return ""
2554             log = extractLogMessageFromGitCommit(next)
2555             settings = extractSettingsGitLog(log)
2556             currentChange = int(settings['change'])
2557             if self.verbose:
2558                 print "current change %s" % currentChange
2559
2560             if currentChange == change:
2561                 if self.verbose:
2562                     print "found %s" % next
2563                 return next
2564
2565             if currentChange < change:
2566                 earliestCommit = "^%s" % next
2567             else:
2568                 latestCommit = "%s" % next
2569
2570         return ""
2571
2572     def importNewBranch(self, branch, maxChange):
2573         # make fast-import flush all changes to disk and update the refs using the checkpoint
2574         # command so that we can try to find the branch parent in the git history
2575         self.gitStream.write("checkpoint\n\n");
2576         self.gitStream.flush();
2577         branchPrefix = self.depotPaths[0] + branch + "/"
2578         range = "@1,%s" % maxChange
2579         #print "prefix" + branchPrefix
2580         changes = p4ChangesForPaths([branchPrefix], range)
2581         if len(changes) <= 0:
2582             return False
2583         firstChange = changes[0]
2584         #print "first change in branch: %s" % firstChange
2585         sourceBranch = self.knownBranches[branch]
2586         sourceDepotPath = self.depotPaths[0] + sourceBranch
2587         sourceRef = self.gitRefForBranch(sourceBranch)
2588         #print "source " + sourceBranch
2589
2590         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2591         #print "branch parent: %s" % branchParentChange
2592         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2593         if len(gitParent) > 0:
2594             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2595             #print "parent git commit: %s" % gitParent
2596
2597         self.importChanges(changes)
2598         return True
2599
2600     def searchParent(self, parent, branch, target):
2601         parentFound = False
2602         for blob in read_pipe_lines(["git", "rev-list", "--reverse", "--no-merges", parent]):
2603             blob = blob.strip()
2604             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2605                 parentFound = True
2606                 if self.verbose:
2607                     print "Found parent of %s in commit %s" % (branch, blob)
2608                 break
2609         if parentFound:
2610             return blob
2611         else:
2612             return None
2613
2614     def importChanges(self, changes):
2615         cnt = 1
2616         for change in changes:
2617             description = p4_describe(change)
2618             self.updateOptionDict(description)
2619
2620             if not self.silent:
2621                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2622                 sys.stdout.flush()
2623             cnt = cnt + 1
2624
2625             try:
2626                 if self.detectBranches:
2627                     branches = self.splitFilesIntoBranches(description)
2628                     for branch in branches.keys():
2629                         ## HACK  --hwn
2630                         branchPrefix = self.depotPaths[0] + branch + "/"
2631                         self.branchPrefixes = [ branchPrefix ]
2632
2633                         parent = ""
2634
2635                         filesForCommit = branches[branch]
2636
2637                         if self.verbose:
2638                             print "branch is %s" % branch
2639
2640                         self.updatedBranches.add(branch)
2641
2642                         if branch not in self.createdBranches:
2643                             self.createdBranches.add(branch)
2644                             parent = self.knownBranches[branch]
2645                             if parent == branch:
2646                                 parent = ""
2647                             else:
2648                                 fullBranch = self.projectName + branch
2649                                 if fullBranch not in self.p4BranchesInGit:
2650                                     if not self.silent:
2651                                         print("\n    Importing new branch %s" % fullBranch);
2652                                     if self.importNewBranch(branch, change - 1):
2653                                         parent = ""
2654                                         self.p4BranchesInGit.append(fullBranch)
2655                                     if not self.silent:
2656                                         print("\n    Resuming with change %s" % change);
2657
2658                                 if self.verbose:
2659                                     print "parent determined through known branches: %s" % parent
2660
2661                         branch = self.gitRefForBranch(branch)
2662                         parent = self.gitRefForBranch(parent)
2663
2664                         if self.verbose:
2665                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2666
2667                         if len(parent) == 0 and branch in self.initialParents:
2668                             parent = self.initialParents[branch]
2669                             del self.initialParents[branch]
2670
2671                         blob = None
2672                         if len(parent) > 0:
2673                             tempBranch = os.path.join(self.tempBranchLocation, "%d" % (change))
2674                             if self.verbose:
2675                                 print "Creating temporary branch: " + tempBranch
2676                             self.commit(description, filesForCommit, tempBranch)
2677                             self.tempBranches.append(tempBranch)
2678                             self.checkpoint()
2679                             blob = self.searchParent(parent, branch, tempBranch)
2680                         if blob:
2681                             self.commit(description, filesForCommit, branch, blob)
2682                         else:
2683                             if self.verbose:
2684                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2685                             self.commit(description, filesForCommit, branch, parent)
2686                 else:
2687                     files = self.extractFilesFromCommit(description)
2688                     self.commit(description, files, self.branch,
2689                                 self.initialParent)
2690                     self.initialParent = ""
2691             except IOError:
2692                 print self.gitError.read()
2693                 sys.exit(1)
2694
2695     def importHeadRevision(self, revision):
2696         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2697
2698         details = {}
2699         details["user"] = "git perforce import user"
2700         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2701                            % (' '.join(self.depotPaths), revision))
2702         details["change"] = revision
2703         newestRevision = 0
2704
2705         fileCnt = 0
2706         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2707
2708         for info in p4CmdList(["files"] + fileArgs):
2709
2710             if 'code' in info and info['code'] == 'error':
2711                 sys.stderr.write("p4 returned an error: %s\n"
2712                                  % info['data'])
2713                 if info['data'].find("must refer to client") >= 0:
2714                     sys.stderr.write("This particular p4 error is misleading.\n")
2715                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2716                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2717                 sys.exit(1)
2718             if 'p4ExitCode' in info:
2719                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2720                 sys.exit(1)
2721
2722
2723             change = int(info["change"])
2724             if change > newestRevision:
2725                 newestRevision = change
2726
2727             if info["action"] in self.delete_actions:
2728                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2729                 #fileCnt = fileCnt + 1
2730                 continue
2731
2732             for prop in ["depotFile", "rev", "action", "type" ]:
2733                 details["%s%s" % (prop, fileCnt)] = info[prop]
2734
2735             fileCnt = fileCnt + 1
2736
2737         details["change"] = newestRevision
2738
2739         # Use time from top-most change so that all git p4 clones of
2740         # the same p4 repo have the same commit SHA1s.
2741         res = p4_describe(newestRevision)
2742         details["time"] = res["time"]
2743
2744         self.updateOptionDict(details)
2745         try:
2746             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2747         except IOError:
2748             print "IO error with git fast-import. Is your git version recent enough?"
2749             print self.gitError.read()
2750
2751
2752     def run(self, args):
2753         self.depotPaths = []
2754         self.changeRange = ""
2755         self.initialParent = ""
2756         self.previousDepotPaths = []
2757
2758         # map from branch depot path to parent branch
2759         self.knownBranches = {}
2760         self.initialParents = {}
2761         self.hasOrigin = originP4BranchesExist()
2762         if not self.syncWithOrigin:
2763             self.hasOrigin = False
2764
2765         if self.importIntoRemotes:
2766             self.refPrefix = "refs/remotes/p4/"
2767         else:
2768             self.refPrefix = "refs/heads/p4/"
2769
2770         if self.syncWithOrigin and self.hasOrigin:
2771             if not self.silent:
2772                 print "Syncing with origin first by calling git fetch origin"
2773             system("git fetch origin")
2774
2775         if len(self.branch) == 0:
2776             self.branch = self.refPrefix + "master"
2777             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2778                 system("git update-ref %s refs/heads/p4" % self.branch)
2779                 system("git branch -D p4");
2780             # create it /after/ importing, when master exists
2781             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
2782                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
2783
2784         # accept either the command-line option, or the configuration variable
2785         if self.useClientSpec:
2786             # will use this after clone to set the variable
2787             self.useClientSpec_from_options = True
2788         else:
2789             if gitConfig("git-p4.useclientspec", "--bool") == "true":
2790                 self.useClientSpec = True
2791         if self.useClientSpec:
2792             self.clientSpecDirs = getClientSpec()
2793
2794         # TODO: should always look at previous commits,
2795         # merge with previous imports, if possible.
2796         if args == []:
2797             if self.hasOrigin:
2798                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2799             self.listExistingP4GitBranches()
2800
2801             if len(self.p4BranchesInGit) > 1:
2802                 if not self.silent:
2803                     print "Importing from/into multiple branches"
2804                 self.detectBranches = True
2805
2806             if self.verbose:
2807                 print "branches: %s" % self.p4BranchesInGit
2808
2809             p4Change = 0
2810             for branch in self.p4BranchesInGit:
2811                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2812
2813                 settings = extractSettingsGitLog(logMsg)
2814
2815                 self.readOptions(settings)
2816                 if (settings.has_key('depot-paths')
2817                     and settings.has_key ('change')):
2818                     change = int(settings['change']) + 1
2819                     p4Change = max(p4Change, change)
2820
2821                     depotPaths = sorted(settings['depot-paths'])
2822                     if self.previousDepotPaths == []:
2823                         self.previousDepotPaths = depotPaths
2824                     else:
2825                         paths = []
2826                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2827                             prev_list = prev.split("/")
2828                             cur_list = cur.split("/")
2829                             for i in range(0, min(len(cur_list), len(prev_list))):
2830                                 if cur_list[i] <> prev_list[i]:
2831                                     i = i - 1
2832                                     break
2833
2834                             paths.append ("/".join(cur_list[:i + 1]))
2835
2836                         self.previousDepotPaths = paths
2837
2838             if p4Change > 0:
2839                 self.depotPaths = sorted(self.previousDepotPaths)
2840                 self.changeRange = "@%s,#head" % p4Change
2841                 if not self.detectBranches:
2842                     self.initialParent = parseRevision(self.branch)
2843                 if not self.silent and not self.detectBranches:
2844                     print "Performing incremental import into %s git branch" % self.branch
2845
2846         if not self.branch.startswith("refs/"):
2847             self.branch = "refs/heads/" + self.branch
2848
2849         if len(args) == 0 and self.depotPaths:
2850             if not self.silent:
2851                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2852         else:
2853             if self.depotPaths and self.depotPaths != args:
2854                 print ("previous import used depot path %s and now %s was specified. "
2855                        "This doesn't work!" % (' '.join (self.depotPaths),
2856                                                ' '.join (args)))
2857                 sys.exit(1)
2858
2859             self.depotPaths = sorted(args)
2860
2861         revision = ""
2862         self.users = {}
2863
2864         # Make sure no revision specifiers are used when --changesfile
2865         # is specified.
2866         bad_changesfile = False
2867         if len(self.changesFile) > 0:
2868             for p in self.depotPaths:
2869                 if p.find("@") >= 0 or p.find("#") >= 0:
2870                     bad_changesfile = True
2871                     break
2872         if bad_changesfile:
2873             die("Option --changesfile is incompatible with revision specifiers")
2874
2875         newPaths = []
2876         for p in self.depotPaths:
2877             if p.find("@") != -1:
2878                 atIdx = p.index("@")
2879                 self.changeRange = p[atIdx:]
2880                 if self.changeRange == "@all":
2881                     self.changeRange = ""
2882                 elif ',' not in self.changeRange:
2883                     revision = self.changeRange
2884                     self.changeRange = ""
2885                 p = p[:atIdx]
2886             elif p.find("#") != -1:
2887                 hashIdx = p.index("#")
2888                 revision = p[hashIdx:]
2889                 p = p[:hashIdx]
2890             elif self.previousDepotPaths == []:
2891                 # pay attention to changesfile, if given, else import
2892                 # the entire p4 tree at the head revision
2893                 if len(self.changesFile) == 0:
2894                     revision = "#head"
2895
2896             p = re.sub ("\.\.\.$", "", p)
2897             if not p.endswith("/"):
2898                 p += "/"
2899
2900             newPaths.append(p)
2901
2902         self.depotPaths = newPaths
2903
2904         # --detect-branches may change this for each branch
2905         self.branchPrefixes = self.depotPaths
2906
2907         self.loadUserMapFromCache()
2908         self.labels = {}
2909         if self.detectLabels:
2910             self.getLabels();
2911
2912         if self.detectBranches:
2913             ## FIXME - what's a P4 projectName ?
2914             self.projectName = self.guessProjectName()
2915
2916             if self.hasOrigin:
2917                 self.getBranchMappingFromGitBranches()
2918             else:
2919                 self.getBranchMapping()
2920             if self.verbose:
2921                 print "p4-git branches: %s" % self.p4BranchesInGit
2922                 print "initial parents: %s" % self.initialParents
2923             for b in self.p4BranchesInGit:
2924                 if b != "master":
2925
2926                     ## FIXME
2927                     b = b[len(self.projectName):]
2928                 self.createdBranches.add(b)
2929
2930         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2931
2932         self.importProcess = subprocess.Popen(["git", "fast-import"],
2933                                               stdin=subprocess.PIPE,
2934                                               stdout=subprocess.PIPE,
2935                                               stderr=subprocess.PIPE);
2936         self.gitOutput = self.importProcess.stdout
2937         self.gitStream = self.importProcess.stdin
2938         self.gitError = self.importProcess.stderr
2939
2940         if revision:
2941             self.importHeadRevision(revision)
2942         else:
2943             changes = []
2944
2945             if len(self.changesFile) > 0:
2946                 output = open(self.changesFile).readlines()
2947                 changeSet = set()
2948                 for line in output:
2949                     changeSet.add(int(line))
2950
2951                 for change in changeSet:
2952                     changes.append(change)
2953
2954                 changes.sort()
2955             else:
2956                 # catch "git p4 sync" with no new branches, in a repo that
2957                 # does not have any existing p4 branches
2958                 if len(args) == 0 and not self.p4BranchesInGit:
2959                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2960                 if self.verbose:
2961                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2962                                                               self.changeRange)
2963                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2964
2965                 if len(self.maxChanges) > 0:
2966                     changes = changes[:min(int(self.maxChanges), len(changes))]
2967
2968             if len(changes) == 0:
2969                 if not self.silent:
2970                     print "No changes to import!"
2971             else:
2972                 if not self.silent and not self.detectBranches:
2973                     print "Import destination: %s" % self.branch
2974
2975                 self.updatedBranches = set()
2976
2977                 self.importChanges(changes)
2978
2979                 if not self.silent:
2980                     print ""
2981                     if len(self.updatedBranches) > 0:
2982                         sys.stdout.write("Updated branches: ")
2983                         for b in self.updatedBranches:
2984                             sys.stdout.write("%s " % b)
2985                         sys.stdout.write("\n")
2986
2987         if gitConfig("git-p4.importLabels", "--bool") == "true":
2988             self.importLabels = True
2989
2990         if self.importLabels:
2991             p4Labels = getP4Labels(self.depotPaths)
2992             gitTags = getGitTags()
2993
2994             missingP4Labels = p4Labels - gitTags
2995             self.importP4Labels(self.gitStream, missingP4Labels)
2996
2997         self.gitStream.close()
2998         if self.importProcess.wait() != 0:
2999             die("fast-import failed: %s" % self.gitError.read())
3000         self.gitOutput.close()
3001         self.gitError.close()
3002
3003         # Cleanup temporary branches created during import
3004         if self.tempBranches != []:
3005             for branch in self.tempBranches:
3006                 read_pipe("git update-ref -d %s" % branch)
3007             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3008
3009         return True
3010
3011 class P4Rebase(Command):
3012     def __init__(self):
3013         Command.__init__(self)
3014         self.options = [
3015                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3016         ]
3017         self.importLabels = False
3018         self.description = ("Fetches the latest revision from perforce and "
3019                             + "rebases the current work (branch) against it")
3020
3021     def run(self, args):
3022         sync = P4Sync()
3023         sync.importLabels = self.importLabels
3024         sync.run([])
3025
3026         return self.rebase()
3027
3028     def rebase(self):
3029         if os.system("git update-index --refresh") != 0:
3030             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3031         if len(read_pipe("git diff-index HEAD --")) > 0:
3032             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
3033
3034         [upstream, settings] = findUpstreamBranchPoint()
3035         if len(upstream) == 0:
3036             die("Cannot find upstream branchpoint for rebase")
3037
3038         # the branchpoint may be p4/foo~3, so strip off the parent
3039         upstream = re.sub("~[0-9]+$", "", upstream)
3040
3041         print "Rebasing the current branch onto %s" % upstream
3042         oldHead = read_pipe("git rev-parse HEAD").strip()
3043         system("git rebase %s" % upstream)
3044         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3045         return True
3046
3047 class P4Clone(P4Sync):
3048     def __init__(self):
3049         P4Sync.__init__(self)
3050         self.description = "Creates a new git repository and imports from Perforce into it"
3051         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3052         self.options += [
3053             optparse.make_option("--destination", dest="cloneDestination",
3054                                  action='store', default=None,
3055                                  help="where to leave result of the clone"),
3056             optparse.make_option("-/", dest="cloneExclude",
3057                                  action="append", type="string",
3058                                  help="exclude depot path"),
3059             optparse.make_option("--bare", dest="cloneBare",
3060                                  action="store_true", default=False),
3061         ]
3062         self.cloneDestination = None
3063         self.needsGit = False
3064         self.cloneBare = False
3065
3066     # This is required for the "append" cloneExclude action
3067     def ensure_value(self, attr, value):
3068         if not hasattr(self, attr) or getattr(self, attr) is None:
3069             setattr(self, attr, value)
3070         return getattr(self, attr)
3071
3072     def defaultDestination(self, args):
3073         ## TODO: use common prefix of args?
3074         depotPath = args[0]
3075         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3076         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3077         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3078         depotDir = re.sub(r"/$", "", depotDir)
3079         return os.path.split(depotDir)[1]
3080
3081     def run(self, args):
3082         if len(args) < 1:
3083             return False
3084
3085         if self.keepRepoPath and not self.cloneDestination:
3086             sys.stderr.write("Must specify destination for --keep-path\n")
3087             sys.exit(1)
3088
3089         depotPaths = args
3090
3091         if not self.cloneDestination and len(depotPaths) > 1:
3092             self.cloneDestination = depotPaths[-1]
3093             depotPaths = depotPaths[:-1]
3094
3095         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3096         for p in depotPaths:
3097             if not p.startswith("//"):
3098                 return False
3099
3100         if not self.cloneDestination:
3101             self.cloneDestination = self.defaultDestination(args)
3102
3103         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3104
3105         if not os.path.exists(self.cloneDestination):
3106             os.makedirs(self.cloneDestination)
3107         chdir(self.cloneDestination)
3108
3109         init_cmd = [ "git", "init" ]
3110         if self.cloneBare:
3111             init_cmd.append("--bare")
3112         subprocess.check_call(init_cmd)
3113
3114         if not P4Sync.run(self, depotPaths):
3115             return False
3116         if self.branch != "master":
3117             if self.importIntoRemotes:
3118                 masterbranch = "refs/remotes/p4/master"
3119             else:
3120                 masterbranch = "refs/heads/p4/master"
3121             if gitBranchExists(masterbranch):
3122                 system("git branch master %s" % masterbranch)
3123                 if not self.cloneBare:
3124                     system("git checkout -f")
3125             else:
3126                 print "Could not detect main branch. No checkout/master branch created."
3127
3128         # auto-set this variable if invoked with --use-client-spec
3129         if self.useClientSpec_from_options:
3130             system("git config --bool git-p4.useclientspec true")
3131
3132         return True
3133
3134 class P4Branches(Command):
3135     def __init__(self):
3136         Command.__init__(self)
3137         self.options = [ ]
3138         self.description = ("Shows the git branches that hold imports and their "
3139                             + "corresponding perforce depot paths")
3140         self.verbose = False
3141
3142     def run(self, args):
3143         if originP4BranchesExist():
3144             createOrUpdateBranchesFromOrigin()
3145
3146         cmdline = "git rev-parse --symbolic "
3147         cmdline += " --remotes"
3148
3149         for line in read_pipe_lines(cmdline):
3150             line = line.strip()
3151
3152             if not line.startswith('p4/') or line == "p4/HEAD":
3153                 continue
3154             branch = line
3155
3156             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3157             settings = extractSettingsGitLog(log)
3158
3159             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3160         return True
3161
3162 class HelpFormatter(optparse.IndentedHelpFormatter):
3163     def __init__(self):
3164         optparse.IndentedHelpFormatter.__init__(self)
3165
3166     def format_description(self, description):
3167         if description:
3168             return description + "\n"
3169         else:
3170             return ""
3171
3172 def printUsage(commands):
3173     print "usage: %s <command> [options]" % sys.argv[0]
3174     print ""
3175     print "valid commands: %s" % ", ".join(commands)
3176     print ""
3177     print "Try %s <command> --help for command specific help." % sys.argv[0]
3178     print ""
3179
3180 commands = {
3181     "debug" : P4Debug,
3182     "submit" : P4Submit,
3183     "commit" : P4Submit,
3184     "sync" : P4Sync,
3185     "rebase" : P4Rebase,
3186     "clone" : P4Clone,
3187     "rollback" : P4RollBack,
3188     "branches" : P4Branches
3189 }
3190
3191
3192 def main():
3193     if len(sys.argv[1:]) == 0:
3194         printUsage(commands.keys())
3195         sys.exit(2)
3196
3197     cmdName = sys.argv[1]
3198     try:
3199         klass = commands[cmdName]
3200         cmd = klass()
3201     except KeyError:
3202         print "unknown command %s" % cmdName
3203         print ""
3204         printUsage(commands.keys())
3205         sys.exit(2)
3206
3207     options = cmd.options
3208     cmd.gitdir = os.environ.get("GIT_DIR", None)
3209
3210     args = sys.argv[2:]
3211
3212     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3213     if cmd.needsGit:
3214         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3215
3216     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3217                                    options,
3218                                    description = cmd.description,
3219                                    formatter = HelpFormatter())
3220
3221     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3222     global verbose
3223     verbose = cmd.verbose
3224     if cmd.needsGit:
3225         if cmd.gitdir == None:
3226             cmd.gitdir = os.path.abspath(".git")
3227             if not isValidGitDir(cmd.gitdir):
3228                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3229                 if os.path.exists(cmd.gitdir):
3230                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3231                     if len(cdup) > 0:
3232                         chdir(cdup);
3233
3234         if not isValidGitDir(cmd.gitdir):
3235             if isValidGitDir(cmd.gitdir + "/.git"):
3236                 cmd.gitdir += "/.git"
3237             else:
3238                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3239
3240         os.environ["GIT_DIR"] = cmd.gitdir
3241
3242     if not cmd.run(args):
3243         parser.print_help()
3244         sys.exit(2)
3245
3246
3247 if __name__ == '__main__':
3248     main()