Revert of Remove SVN and File support from gclient (patchset #4 id:60001 of https://codereview.chromium.org/2398493004/ )

Reason for revert:
The Chrome buildbots pass "--transitive" still. Removing the option causes them to fail to build.

Example:

https://uberchromegw.corp.google.com/i/chromeos.chrome/builders/x86-alex-tot-chrome-pfq-informational/builds/21555

Original issue's description:
> Remove SVN and File support from gclient
>
> FileImpl was only usable with SVN.
>
> R=maruel@chromium.org
> BUG=641588
>
> Committed: b3a24c3fd5

TBR=maruel@chromium.org,agable@chromium.org
# Skipping CQ checks because original CL landed less than 1 days ago.
NOPRESUBMIT=true
NOTREECHECKS=true
NOTRY=true
BUG=641588

Review-Url: https://codereview.chromium.org/2405173003

gclient.py

@@ -3,7 +3,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Meta checkout dependency manager for Git."""
+"""Meta checkout manager supporting both Subversion and GIT."""
 # Files
 #   .gclient      : Current client configuration, written by 'config' command.
 #                   Format is a Python script defining 'solutions', a list whose
@@ -173,6 +173,29 @@ class GClientKeywords(object):
       return 'From(%s, %s)' % (repr(self.module_name),
                                repr(self.sub_target_name))
 
+  class FileImpl(object):
+    """Used to implement the File('') syntax which lets you sync a single file
+    from a SVN repo."""
+
+    def __init__(self, file_location):
+      self.file_location = file_location
+
+    def __str__(self):
+      return 'File("%s")' % self.file_location
+
+    def GetPath(self):
+      return os.path.split(self.file_location)[0]
+
+    def GetFilename(self):
+      rev_tokens = self.file_location.split('@')
+      return os.path.split(rev_tokens[0])[1]
+
+    def GetRevision(self):
+      rev_tokens = self.file_location.split('@')
+      if len(rev_tokens) > 1:
+        return rev_tokens[1]
+      return None
+
   class VarImpl(object):
     def __init__(self, custom_vars, local_scope):
       self._custom_vars = custom_vars
@@ -221,15 +244,24 @@ class DependencySettings(GClientKeywords):
     self._custom_deps = custom_deps or {}
     self._custom_hooks = custom_hooks or []
 
+    # TODO(iannucci): Remove this when all masters are correctly substituting
+    # the new blink url.
+    if (self._custom_vars.get('webkit_trunk', '') ==
+        'svn://svn-mirror.golo.chromium.org/webkit-readonly/trunk'):
+      new_url = 'svn://svn-mirror.golo.chromium.org/blink/trunk'
+      print('Overwriting Var("webkit_trunk") with %s' % new_url)
+      self._custom_vars['webkit_trunk'] = new_url
+
     # Post process the url to remove trailing slashes.
     if isinstance(self._url, basestring):
       # urls are sometime incorrectly written as proto://host/path/@rev. Replace
       # it to proto://host/path@rev.
       self._url = self._url.replace('/@', '@')
-    elif not isinstance(self._url, (self.FromImpl, None.__class__)):
+    elif not isinstance(self._url,
+        (self.FromImpl, self.FileImpl, None.__class__)):
       raise gclient_utils.Error(
           ('dependency url must be either a string, None, '
-           'or From() instead of %s') % self._url.__class__.__name__)
+           'File() or From() instead of %s') % self._url.__class__.__name__)
     # Make any deps_file path platform-appropriate.
     for sep in ['/', '\\']:
       self._deps_file = self._deps_file.replace(sep, os.sep)
@@ -502,6 +534,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
               'relative DEPS entry \'%s\' must begin with a slash' % url)
         # Create a scm just to query the full url.
         parent_url = self.parent.parsed_url
+        if isinstance(parent_url, self.FileImpl):
+          parent_url = parent_url.file_location
         scm = gclient_scm.CreateSCM(
             parent_url, self.root.root_dir, None, self.outbuf)
         parsed_url = scm.FullUrlForRelativeUrl(url)
@@ -512,6 +546,12 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
           (self.name, url, parsed_url))
       return parsed_url
 
+    if isinstance(url, self.FileImpl):
+      logging.info(
+          'Dependency(%s).LateOverride(%s) -> %s (File)' %
+          (self.name, url, url))
+      return url
+
     if url is None:
       logging.info(
           'Dependency(%s).LateOverride(%s) -> %s' % (self.name, url, url))
@@ -602,6 +642,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
         }
       else:
         global_scope = {
+          'File': self.FileImpl,
           'From': self.FromImpl,
           'Var': var.Lookup,
           'deps_os': {},
@@ -732,6 +773,41 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
         self.add_dependency(dep)
     self._mark_as_parsed(hooks)
 
+  def maybeGetParentRevision(self, command, options, parsed_url, parent):
+    """Uses revision/timestamp of parent if no explicit revision was specified.
+
+    If we are performing an update and --transitive is set, use
+    - the parent's revision if 'self.url' is in the same repository
+    - the parent's timestamp otherwise
+    to update 'self.url'. The used revision/timestamp will be set in
+    'options.revision'.
+    If we have an explicit revision do nothing.
+    """
+    if command == 'update' and options.transitive and not options.revision:
+      _, revision = gclient_utils.SplitUrlRevision(parsed_url)
+      if not revision:
+        options.revision = getattr(parent, '_used_revision', None)
+        if (options.revision and
+            not gclient_utils.IsDateRevision(options.revision)):
+          assert self.parent and self.parent.used_scm
+          # If this dependency is in the same repository as parent it's url will
+          # start with a slash. If so we take the parent revision instead of
+          # it's timestamp.
+          # (The timestamps of commits in google code are broken -- which can
+          # result in dependencies to be checked out at the wrong revision)
+          if self.url.startswith('/'):
+            if options.verbose:
+              print('Using parent\'s revision %s since we are in the same '
+                    'repository.' % options.revision)
+          else:
+            parent_revision_date = self.parent.used_scm.GetRevisionDate(
+                options.revision)
+            options.revision = gclient_utils.MakeDateRevision(
+                parent_revision_date)
+            if options.verbose:
+              print('Using parent\'s revision date %s since we are in a '
+                    'different repository.' % options.revision)
+
   def findDepsFromNotAllowedHosts(self):
     """Returns a list of depenecies from not allowed hosts.
 
@@ -766,9 +842,23 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     file_list = [] if not options.nohooks else None
     revision_override = revision_overrides.pop(self.name, None)
     if run_scm and parsed_url:
+      if isinstance(parsed_url, self.FileImpl):
+        # Special support for single-file checkout.
+        if not command in (None, 'cleanup', 'diff', 'pack', 'status'):
+          # Sadly, pylint doesn't realize that parsed_url is of FileImpl.
+          # pylint: disable=E1103
+          options.revision = parsed_url.GetRevision()
+          self._used_scm = gclient_scm.SVNWrapper(
+              parsed_url.GetPath(), self.root.root_dir, self.name,
+              out_cb=work_queue.out_cb)
+          self._used_scm.RunCommand('updatesingle',
+              options, args + [parsed_url.GetFilename()], file_list)
+      else:
         # Create a shallow copy to mutate revision.
         options = copy.copy(options)
         options.revision = revision_override
+        self.maybeGetParentRevision(
+            command, options, parsed_url, self.parent)
         self._used_revision = options.revision
         self._used_scm = gclient_scm.CreateSCM(
             parsed_url, self.root.root_dir, self.name, self.outbuf,
@@ -803,6 +893,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
         work_queue.enqueue(s)
 
     if command == 'recurse':
+      if not isinstance(parsed_url, self.FileImpl):
         # Skip file only checkout.
         scm = gclient_scm.GetScmName(parsed_url)
         if not options.scm or scm in options.scm:
@@ -894,10 +985,11 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     # If "--force" was specified, run all hooks regardless of what files have
     # changed.
     if self.deps_hooks:
-      # TODO(maruel): If the user is using git, then we don't know
+      # TODO(maruel): If the user is using git or git-svn, then we don't know
       # what files have changed so we always run all hooks. It'd be nice to fix
       # that.
       if (options.force or
+          isinstance(self.parsed_url, self.FileImpl) or
           gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
           os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))):
         for hook_dict in self.deps_hooks:
@@ -1181,7 +1273,9 @@ The local checkout in %(checkout_path)s reports:
 %(actual_url)s (%(actual_scm)s)
 
 You should ensure that the URL listed in .gclient is correct and either change
-it or fix the checkout.
+it or fix the checkout. If you're managing your own git checkout in
+%(checkout_path)s but the URL in .gclient is for an svn repository, you probably
+want to set 'managed': False in .gclient.
 '''  % {'checkout_path': os.path.join(self.root_dir, dep.name),
         'expected_url': dep.url,
         'expected_scm': gclient_scm.GetScmName(dep.url),
@@ -1244,6 +1338,82 @@ it or fix the checkout.
                                          self._options.config_filename),
                             self.config_content)
 
+  def MigrateConfigToGit(self, path, options):
+    svn_url_re = re.compile('^(https?://src\.chromium\.org/svn|'
+                            'svn://svn\.chromium\.org/chrome)/'
+                            '(trunk|branches/[^/]+)/src')
+    old_git_re = re.compile('^(https?://git\.chromium\.org|'
+                            'ssh://([a-zA-Z_][a-zA-Z0-9_-]*@)?'
+                            'gerrit\.chromium\.org(:2941[89])?)/'
+                            'chromium/src\.git')
+    # Scan existing .gclient file for obsolete settings.  It would be simpler
+    # to traverse self.dependencies, but working with the AST allows the code to
+    # dump an updated .gclient file that preserves the ordering of the original.
+    a = ast.parse(self.config_content, options.config_filename, 'exec')
+    modified = False
+    solutions = [elem for elem in a.body if 'solutions' in
+                 [target.id for target in elem.targets]]
+    if not solutions:
+      return self
+    solutions = solutions[-1]
+    for solution in solutions.value.elts:
+      # Check for obsolete URL's
+      url_idx = ast_dict_index(solution, 'url')
+      if url_idx == -1:
+        continue
+      url_val = solution.values[url_idx]
+      if type(url_val) is not ast.Str:
+        continue
+      if (svn_url_re.match(url_val.s.strip())):
+        raise gclient_utils.Error(
+"""
+The chromium code repository has migrated completely to git.
+Your SVN-based checkout is now obsolete; you need to create a brand-new
+git checkout by following these instructions:
+
+http://www.chromium.org/developers/how-tos/get-the-code
+""")
+      if (old_git_re.match(url_val.s.strip())):
+        url_val.s = CHROMIUM_SRC_URL
+        modified = True
+
+      # Ensure deps_file is set to .DEPS.git.  We enforce this here to smooth
+      # over switching between pre-git-migration and post-git-migration
+      # revisions.
+      #   - For pre-migration revisions, .DEPS.git must be explicitly set.
+      #   - For post-migration revisions, .DEPS.git is not present, so gclient
+      #     will correctly fall back to DEPS.
+      if url_val.s == CHROMIUM_SRC_URL:
+        deps_file_idx = ast_dict_index(solution, 'deps_file')
+        if deps_file_idx != -1:
+          continue
+        solution.keys.append(ast.Str('deps_file'))
+        solution.values.append(ast.Str('.DEPS.git'))
+        modified = True
+
+    if not modified:
+      return self
+
+    print(
+"""
+WARNING: gclient detected an obsolete setting in your %s file.  The file has
+been automagically updated.  The previous version is available at %s.old.
+""" % (options.config_filename, options.config_filename))
+
+    # Replace existing .gclient with the updated version.
+    # Return a new GClient instance based on the new content.
+    new_content = ast2str(a)
+    dot_gclient_fn = os.path.join(path, options.config_filename)
+    try:
+      os.rename(dot_gclient_fn, dot_gclient_fn + '.old')
+    except OSError:
+      pass
+    with open(dot_gclient_fn, 'w') as fh:
+      fh.write(new_content)
+    client = GClient(path, options)
+    client.SetConfig(new_content)
+    return client
+
   @staticmethod
   def LoadCurrentConfig(options):
     """Searches for and loads a .gclient file relative to the current working
@@ -1261,6 +1431,7 @@ it or fix the checkout.
       client = GClient(path, options)
       client.SetConfig(gclient_utils.FileRead(
           os.path.join(path, options.config_filename)))
+      client = client.MigrateConfigToGit(path, options)
 
     if (options.revisions and
         len(client.dependencies) > 1 and
@@ -1295,6 +1466,8 @@ it or fix the checkout.
     # makes testing a bit too fun.
     result = 'entries = {\n'
     for entry in self.root.subtree(False):
+      # Skip over File() dependencies as we can't version them.
+      if not isinstance(entry.parsed_url, self.FileImpl):
         result += '  %s: %s,\n' % (pprint.pformat(entry.name),
             pprint.pformat(entry.parsed_url))
     result += '}\n'
@@ -1426,12 +1599,17 @@ it or fix the checkout.
               prev_url, self.root_dir, entry_fixed, self.outbuf)
 
           # Check to see if this directory is now part of a higher-up checkout.
+          # The directory might be part of a git OR svn checkout.
           scm_root = None
+          scm_class = None
+          for scm_class in (gclient_scm.scm.GIT, gclient_scm.scm.SVN):
             try:
-            scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(scm.checkout_path)
+              scm_root = scm_class.GetCheckoutRoot(scm.checkout_path)
             except subprocess2.CalledProcessError:
               pass
-          if not scm_root:
+            if scm_root:
+              break
+          else:
             logging.warning('Could not find checkout root for %s. Unable to '
                             'determine whether it is part of a higher-level '
                             'checkout, so not removing.' % entry)
@@ -1442,14 +1620,14 @@ it or fix the checkout.
           # If the subproject is a Git project, we need to remove its .git
           # folder. Otherwise git operations on that folder will have different
           # effects depending on the current working directory.
-          if os.path.abspath(scm_root) == os.path.abspath(e_dir):
+          if scm_class == gclient_scm.scm.GIT and (
+              os.path.abspath(scm_root) == os.path.abspath(e_dir)):
             e_par_dir = os.path.join(e_dir, os.pardir)
-            if gclient_scm.scm.GIT.IsInsideWorkTree(e_par_dir):
+            if scm_class.IsInsideWorkTree(e_par_dir):
-              par_scm_root = gclient_scm.scm.GIT.GetCheckoutRoot(e_par_dir)
+              par_scm_root = scm_class.GetCheckoutRoot(e_par_dir)
               # rel_e_dir : relative path of entry w.r.t. its parent repo.
               rel_e_dir = os.path.relpath(e_dir, par_scm_root)
-              if gclient_scm.scm.GIT.IsDirectoryVersioned(
+              if scm_class.IsDirectoryVersioned(par_scm_root, rel_e_dir):
-                  par_scm_root, rel_e_dir):
                 save_dir = scm.GetGitBackupDirPath()
                 # Remove any eventual stale backup dir for the same project.
                 if os.path.exists(save_dir):
@@ -1462,7 +1640,7 @@ it or fix the checkout.
                 # In such case we want to cleanup any eventual stale files
                 # (coming from the old subproject) in order to end up with a
                 # clean checkout.
-                gclient_scm.scm.GIT.CleanupDir(par_scm_root, rel_e_dir)
+                scm_class.CleanupDir(par_scm_root, rel_e_dir)
                 assert not os.path.exists(os.path.join(e_dir, '.git'))
                 print(('\nWARNING: \'%s\' has been moved from DEPS to a higher '
                        'level checkout. The git folder containing all the local'
@@ -1510,9 +1688,13 @@ it or fix the checkout.
       """Returns the revision-qualified SCM url for a Dependency."""
       if dep.parsed_url is None:
         return None
-      url, _ = gclient_utils.SplitUrlRevision(dep.parsed_url)
+      if isinstance(dep.parsed_url, self.FileImpl):
+        original_url = dep.parsed_url.file_location
+      else:
+        original_url = dep.parsed_url
+      url, _ = gclient_utils.SplitUrlRevision(original_url)
       scm = gclient_scm.CreateSCM(
-          dep.parsed_url, self.root_dir, dep.name, self.outbuf)
+          original_url, self.root_dir, dep.name, self.outbuf)
       if not os.path.isdir(scm.checkout_path):
         return None
       return '%s@%s' % (url, scm.revinfo(self._options, [], None))
@@ -1596,9 +1778,9 @@ it or fix the checkout.
 
 
 def CMDcleanup(parser, args):
-  """DEPRECATED: SVN-only. Cleaned up all working copies.
+  """Cleans up all working copies.
 
-  This is a no-op in Git.
+  Mostly svn-specific. Simply runs 'svn cleanup' for each module.
   """
   parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
                     help='override deps for the specified (comma-separated) '
@@ -1781,7 +1963,7 @@ def CMDconfig(parser, args):
 def CMDpack(parser, args):
   """Generates a patch which can be applied at the root of the tree.
 
-  Internally, runs 'git diff' on each checked out module and
+  Internally, runs 'svn diff'/'git diff' on each checked out module and
   dependencies, and performs minimal postprocessing of the output. The
   resulting patch is printed to stdout and can be applied to a freshly
   checked out tree via 'patch -p0 < patchfile'.
@@ -1836,8 +2018,8 @@ os_deps, etc.)
 {
   "solutions" : {
     "<name>": {  # <name> is the posix-normalized path to the solution.
-      "revision": [<git id hex string>|null],
+      "revision": [<svn rev int>|<git id hex string>|null],
-      "scm": ["git"|null],
+      "scm": ["svn"|"git"|null],
     }
   }
 }
@@ -1864,6 +2046,11 @@ def CMDsync(parser, args):
                          'full checkout. (git only)')
   parser.add_option('--with_tags', action='store_true',
                     help='Clone git tags in addition to the default refspecs.')
+  parser.add_option('-t', '--transitive', action='store_true',
+                    help='When a revision is specified (in the DEPS file or '
+                          'with the command-line flag), transitively update '
+                          'the dependencies to the date of the given revision. '
+                          'Only supported for SVN repositories.')
   parser.add_option('-H', '--head', action='store_true',
                     help='skips any safesync_urls specified in '
                          'configured solutions and sync to head instead')
@@ -1888,6 +2075,9 @@ def CMDsync(parser, args):
                     help='override deps for the specified (comma-separated) '
                          'platform(s); \'all\' will process all deps_os '
                          'references')
+  parser.add_option('-m', '--manually_grab_svn_rev', action='store_true',
+                    help='Skip svn up whenever possible by requesting '
+                         'actual HEAD revision from the repository')
   parser.add_option('--upstream', action='store_true',
                     help='Make repo state match upstream branch.')
   parser.add_option('--output-json',
@@ -1961,7 +2151,7 @@ def CMDrevert(parser, args):
   """Reverts all modifications in every dependencies.
 
   That's the nuclear option to get back to a 'clean' state. It removes anything
-  that shows up in git status."""
+  that shows up in svn status."""
   parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
                     help='override deps for the specified (comma-separated) '
                          'platform(s); \'all\' will process all deps_os '
@@ -2013,8 +2203,9 @@ def CMDrevinfo(parser, args):
 
   This allows the capture of an overall 'revision' for the source tree that
   can be used to reproduce the same tree in the future. It is only useful for
-  'unpinned dependencies', i.e. DEPS/deps references without a git hash.
+  'unpinned dependencies', i.e. DEPS/deps references without a svn revision
-  A git branch name isn't 'pinned' since the actual commit can change.
+  number or a git hash. A git branch name isn't 'pinned' since the actual
+  commit can change.
   """
   parser.add_option('--deps', dest='deps_os', metavar='OS_LIST',
                     help='override deps for the specified (comma-separated) '
@@ -2080,6 +2271,12 @@ class OptionParser(optparse.OptionParser):
       jobs = 1
     else:
       jobs = max(8, gclient_utils.NumLocalCpus())
+    # cmp: 2013/06/19
+    # Temporary workaround to lower bot-load on SVN server.
+    # Bypassed if a bot_update flag is detected.
+    if (os.environ.get('CHROME_HEADLESS') == '1' and
+        not os.path.exists('update.flag')):
+      jobs = 1
 
     self.add_option(
         '-j', '--jobs', default=jobs, type='int',
@@ -2130,6 +2327,8 @@ class OptionParser(optparse.OptionParser):
       options.noprehooks = True
     if not hasattr(options, 'deps_os'):
       options.deps_os = None
+    if not hasattr(options, 'manually_grab_svn_rev'):
+      options.manually_grab_svn_rev = None
     if not hasattr(options, 'force'):
       options.force = None
     return (options, args)

gclient_scm.py

@@ -392,6 +392,14 @@ class GitWrapper(SCMWrapper):
     if managed:
       self._DisableHooks()
 
+    if gclient_utils.IsDateRevision(revision):
+      # Date-revisions only work on git-repositories if the reflog hasn't
+      # expired yet. Use rev-list to get the corresponding revision.
+      #  git rev-list -n 1 --before='time-stamp' branchname
+      if options.transitive:
+        self.Print('Warning: --transitive only works for SVN repositories.')
+        revision = default_rev
+
     rev_str = ' at %s' % revision
     files = [] if file_list is not None else None
 
@@ -1484,6 +1492,12 @@ class SVNWrapper(SCMWrapper):
                 self.Print('You can pass --force to enable automatic removal.')
               raise e
 
+    # Retrieve the current HEAD version because svn is slow at null updates.
+    if options.manually_grab_svn_rev and not revision:
+      from_info_live = scm.SVN.CaptureRemoteInfo(from_info['URL'])
+      revision = str(from_info_live['Revision'])
+      rev_str = ' at %s' % revision
+
     if from_info['URL'].rstrip('/') != base_url.rstrip('/'):
       # The repository url changed, need to switch.
       try:
@@ -1713,6 +1727,10 @@ class SVNWrapper(SCMWrapper):
       new_command.append('--force')
       if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
         new_command.extend(('--accept', 'theirs-conflict'))
+    elif options.manually_grab_svn_rev:
+      new_command.append('--force')
+      if command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
+        new_command.extend(('--accept', 'postpone'))
     elif command[0] != 'checkout' and scm.SVN.AssertVersion('1.6')[0]:
       new_command.extend(('--accept', 'postpone'))
     return new_command

tests/gclient_test.py

@@ -244,13 +244,17 @@ class GclientTest(trial_dir.TestCase):
           obj.dependencies[0], 'foo/dir2',
           gclient.GClientKeywords.FromImpl('bar'), None, None, None, None,
           None, 'DEPS', True, False),
+        gclient.Dependency(
+          obj.dependencies[0], 'foo/dir3',
+          gclient.GClientKeywords.FileImpl('url'), None, None, None, None,
+          None, 'DEPS', True, False),
       ],
       [])
     # Make sure __str__() works fine.
     # pylint: disable=W0212
     obj.dependencies[0]._file_list.append('foo')
     str_obj = str(obj)
-    self.assertEquals(370, len(str_obj), '%d\n%s' % (len(str_obj), str_obj))
+    self.assertEquals(471, len(str_obj), '%d\n%s' % (len(str_obj), str_obj))
 
   def testHooks(self):
     topdir = self.root_dir