Add cipd support to gclient.

Bug: 789809
Change-Id: I9942eaa613d51edd77ee5195603932a103f5e3cd
Reviewed-on: https://chromium-review.googlesource.com/829953
Commit-Queue: John Budorick <jbudorick@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>

gclient.py

@@ -277,8 +277,9 @@ class DependencySettings(object):
           ('dependency url must be either string or None, '
            'instead of %s') % self._url.__class__.__name__)
     # Make any deps_file path platform-appropriate.
-    for sep in ['/', '\\']:
-      self._deps_file = self._deps_file.replace(sep, os.sep)
+    if self._deps_file:
+      for sep in ['/', '\\']:
+        self._deps_file = self._deps_file.replace(sep, os.sep)
 
   @property
   def deps_file(self):
@@ -422,6 +423,22 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
     if not self.name and self.parent:
       raise gclient_utils.Error('Dependency without name')
 
+  def ToLines(self):
+    s = []
+    condition_part = (['    "condition": %r,' % self.condition]
+                      if self.condition else [])
+    s.extend([
+        '  # %s' % self.hierarchy(include_url=False),
+        '  "%s": {' % (self.name,),
+        '    "url": "%s",' % (self.raw_url,),
+    ] + condition_part + [
+        '  },',
+        '',
+    ])
+    return s
+
+
+
   @property
   def requirements(self):
     """Calculate the list of requirements."""
@@ -534,7 +551,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
               'relative DEPS entry \'%s\' must begin with a slash' % url)
         # Create a scm just to query the full url.
         parent_url = self.parent.parsed_url
-        scm = gclient_scm.CreateSCM(
+        scm = self.CreateSCM(
             parent_url, self.root.root_dir, None, self.outbuf)
         parsed_url = scm.FullUrlForRelativeUrl(url)
       else:
@@ -623,6 +640,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
   def _deps_to_objects(self, deps, use_relative_paths):
     """Convert a deps dict to a dict of Dependency objects."""
     deps_to_add = []
+    cipd_root = None
     for name, dep_value in deps.iteritems():
       should_process = self.recursion_limit and self.should_process
       deps_file = self.deps_file
@@ -632,30 +650,58 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
           deps_file = ent['deps_file']
       if dep_value is None:
         continue
+
       condition = None
       condition_value = True
       if isinstance(dep_value, basestring):
         raw_url = dep_value
+        dep_type = None
       else:
         # This should be guaranteed by schema checking in gclient_eval.
         assert isinstance(dep_value, collections.Mapping)
-        raw_url = dep_value['url']
+        raw_url = dep_value.get('url')
         # Take into account should_process metadata set by MergeWithOsDeps.
         should_process = (should_process and
                           dep_value.get('should_process', True))
         condition = dep_value.get('condition')
-
-      url = raw_url.format(**self.get_vars())
+        dep_type = dep_value.get('dep_type')
 
       if condition:
         condition_value = gclient_eval.EvaluateCondition(
             condition, self.get_vars())
         if not self._get_option('process_all_deps', False):
           should_process = should_process and condition_value
-      deps_to_add.append(Dependency(
-          self, name, raw_url, url, None, None, self.custom_vars, None,
-          deps_file, should_process, use_relative_paths, condition,
-          condition_value))
+
+      if dep_type == 'cipd':
+        if not cipd_root:
+          cipd_root = gclient_scm.CipdRoot(
+              os.path.join(self.root.root_dir, self.name),
+              # TODO(jbudorick): Support other service URLs as necessary.
+              # Service URLs should be constant over the scope of a cipd
+              # root, so a var per DEPS file specifying the service URL
+              # should suffice.
+              'https://chrome-infra-packages.appspot.com')
+        for package in dep_value.get('packages', []):
+          deps_to_add.append(
+              CipdDependency(
+                  self, name, package, cipd_root,
+                  self.custom_vars, should_process, use_relative_paths,
+                  condition, condition_value))
+      elif dep_type == 'git':
+        url = raw_url.format(**self.get_vars())
+        deps_to_add.append(
+            GitDependency(
+                self, name, raw_url, url, None, None, self.custom_vars, None,
+                deps_file, should_process, use_relative_paths, condition,
+                condition_value))
+      else:
+        url = raw_url.format(**self.get_vars())
+        deps_to_add.append(
+            Dependency(
+                self, name, raw_url, url, None, None, self.custom_vars, None,
+                deps_file, should_process, use_relative_paths, condition,
+                condition_value))
+
     deps_to_add.sort(key=lambda x: x.name)
     return deps_to_add
 
@@ -695,7 +741,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
       # Eval the content.
       try:
         if self._get_option('validate_syntax', False):
-          gclient_eval.Exec(deps_content, global_scope, local_scope, filepath)
+          local_scope = gclient_eval.Exec(
+              deps_content, global_scope, local_scope, filepath)
         else:
           exec(deps_content, global_scope, local_scope)
       except SyntaxError as e:
@@ -878,7 +925,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
       options = copy.copy(options)
       options.revision = revision_override
       self._used_revision = options.revision
-      self._used_scm = gclient_scm.CreateSCM(
+      self._used_scm = self.CreateSCM(
           parsed_url, self.root.root_dir, self.name, self.outbuf,
           out_cb=work_queue.out_cb)
       self._got_revision = self._used_scm.RunCommand(command, options, args,
@@ -913,7 +960,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
 
     if command == 'recurse':
       # Skip file only checkout.
-      scm = gclient_scm.GetScmName(parsed_url)
+      scm = self.GetScmName(parsed_url)
       if not options.scm or scm in options.scm:
         cwd = os.path.normpath(os.path.join(self.root.root_dir, self.name))
         # Pass in the SCM type as an env variable.  Make sure we don't put
@@ -967,6 +1014,40 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
         else:
           print('Skipped missing %s' % cwd, file=sys.stderr)
 
+  def GetScmName(self, url):
+    """Get the name of the SCM for the given URL.
+
+    While we currently support both git and cipd as SCM implementations,
+    this currently cannot return 'cipd', regardless of the URL, as CIPD
+    has no canonical URL format. If you want to use CIPD as an SCM, you
+    must currently do so by explicitly using a CipdDependency.
+    """
+    if not url:
+      return None
+    url, _ = gclient_utils.SplitUrlRevision(url)
+    if url.endswith('.git'):
+      return 'git'
+    protocol = url.split('://')[0]
+    if protocol in (
+        'file', 'git', 'git+http', 'git+https', 'http', 'https', 'ssh', 'sso'):
+      return 'git'
+    return None
+
+  def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
+                out_cb=None):
+    SCM_MAP = {
+      'cipd': gclient_scm.CipdWrapper,
+      'git': gclient_scm.GitWrapper,
+    }
+
+    scm_name = self.GetScmName(url)
+    if not scm_name in SCM_MAP:
+      raise gclient_utils.Error('No SCM found for url %s' % url)
+    scm_class = SCM_MAP[scm_name]
+    if not scm_class.BinaryExists():
+      raise gclient_utils.Error('%s command not found' % scm_name)
+    return scm_class(url, root_dir, relpath, out_fh, out_cb)
+
   def HasGNArgsFile(self):
     return self._gn_args_file is not None
 
@@ -1004,7 +1085,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
       # what files have changed so we always run all hooks. It'd be nice to fix
       # that.
       if (options.force or
-          gclient_scm.GetScmName(self.parsed_url) in ('git', None) or
+          self.GetScmName(self.parsed_url) in ('git', None) or
           os.path.isdir(os.path.join(self.root.root_dir, self.name, '.git'))):
         result.extend(self.deps_hooks)
       else:
@@ -1280,7 +1361,7 @@ solutions = [
     solutions."""
     for dep in self.dependencies:
       if dep.managed and dep.url:
-        scm = gclient_scm.CreateSCM(
+        scm = self.CreateSCM(
             dep.url, self.root_dir, dep.name, self.outbuf)
         actual_url = scm.GetActualRemoteURL(self._options)
         if actual_url and not scm.DoesRemoteURLMatch(self._options):
@@ -1305,10 +1386,10 @@ You should ensure that the URL listed in .gclient is correct and either change
 it or fix the checkout.
 '''  % {'checkout_path': os.path.join(self.root_dir, dep.name),
         'expected_url': dep.url,
-        'expected_scm': gclient_scm.GetScmName(dep.url),
+        'expected_scm': self.GetScmName(dep.url),
         'mirror_string' : mirror_string,
         'actual_url': actual_url,
-        'actual_scm': gclient_scm.GetScmName(actual_url)})
+        'actual_scm': self.GetScmName(actual_url)})
 
   def SetConfig(self, content):
     assert not self.dependencies
@@ -1529,7 +1610,7 @@ it or fix the checkout.
             (not any(path.startswith(entry + '/') for path in entries)) and
             os.path.exists(e_dir)):
           # The entry has been removed from DEPS.
-          scm = gclient_scm.CreateSCM(
+          scm = self.CreateSCM(
               prev_url, self.root_dir, entry_fixed, self.outbuf)
 
           # Check to see if this directory is now part of a higher-up checkout.
@@ -1619,7 +1700,7 @@ it or fix the checkout.
       if dep.parsed_url is None:
         return None
       url, _ = gclient_utils.SplitUrlRevision(dep.parsed_url)
-      scm = gclient_scm.CreateSCM(
+      scm = dep.CreateSCM(
           dep.parsed_url, self.root_dir, dep.name, self.outbuf)
       if not os.path.isdir(scm.checkout_path):
         return None
@@ -1699,6 +1780,91 @@ it or fix the checkout.
     return self._enforced_os
 
 
+class GitDependency(Dependency):
+  """A Dependency object that represents a single git checkout."""
+
+  #override
+  def GetScmName(self, url):
+    """Always 'git'."""
+    del url
+    return 'git'
+
+  #override
+  def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
+                out_cb=None):
+    """Create a Wrapper instance suitable for handling this git dependency."""
+    return gclient_scm.GitWrapper(url, root_dir, relpath, out_fh, out_cb)
+
+
+class CipdDependency(Dependency):
+  """A Dependency object that represents a single CIPD package."""
+
+  def __init__(
+      self, parent, name, dep_value, cipd_root,
+      custom_vars, should_process, relative, condition, condition_value):
+    package = dep_value['package']
+    version = dep_value['version']
+    url = urlparse.urljoin(
+        cipd_root.service_url, '%s@%s' % (package, version))
+    super(CipdDependency, self).__init__(
+        parent, name, url, url, None, None, custom_vars,
+        None, None, should_process, relative, condition, condition_value)
+    if relative:
+      # TODO(jbudorick): Implement relative if necessary.
+      raise gclient_utils.Error(
+          'Relative CIPD dependencies are not currently supported.')
+    self._cipd_root = cipd_root
+
+    self._cipd_subdir = os.path.relpath(
+        os.path.join(self.root.root_dir, self.name), cipd_root.root_dir)
+    self._cipd_package = self._cipd_root.add_package(
+        self._cipd_subdir, package, version)
+
+  def ParseDepsFile(self):
+    """CIPD dependencies are not currently allowed to have nested deps."""
+    self.add_dependencies_and_close([], [])
+
+  #override
+  def GetScmName(self, url):
+    """Always 'cipd'."""
+    del url
+    return 'cipd'
+
+  #override
+  def CreateSCM(self, url, root_dir=None, relpath=None, out_fh=None,
+                out_cb=None):
+    """Create a Wrapper instance suitable for handling this CIPD dependency."""
+    return gclient_scm.CipdWrapper(
+        url, root_dir, relpath, out_fh, out_cb,
+        root=self._cipd_root,
+        package=self._cipd_package)
+
+  def ToLines(self):
+    """Return a list of lines representing this in a DEPS file."""
+    s = []
+    if self._cipd_package.authority_for_subdir:
+      condition_part = (['    "condition": %r,' % self.condition]
+                        if self.condition else [])
+      s.extend([
+          '  # %s' % self.hierarchy(include_url=False),
+          '  "%s": {' % (self.name,),
+          '    "packages": [',
+      ])
+      for p in self._cipd_root.packages(self._cipd_subdir):
+        s.extend([
+            '      "package": "%s",' % p.name,
+            '      "version": "%s",' % p.version,
+        ])
+      s.extend([
+          '    ],',
+          '    "dep_type": "cipd",',
+      ] + condition_part + [
+          '  },',
+          '',
+      ])
+    return s
+
+
 #### gclient commands.
 
 
@@ -1809,7 +1975,7 @@ class Flattener(object):
     if revision and gclient_utils.IsFullGitSha(revision):
       return
 
-    scm = gclient_scm.CreateSCM(
+    scm = dep.CreateSCM(
         dep.parsed_url, self._client.root_dir, dep.name, dep.outbuf)
     revinfo = scm.revinfo(self._client._options, [], None)
 
@@ -2028,17 +2194,8 @@ def _DepsToLines(deps):
   if not deps:
     return []
   s = ['deps = {']
-  for name, dep in sorted(deps.iteritems()):
-    condition_part = (['    "condition": %r,' % dep.condition]
-                      if dep.condition else [])
-    s.extend([
-        '  # %s' % dep.hierarchy(include_url=False),
-        '  "%s": {' % (name,),
-        '    "url": "%s",' % (dep.raw_url,),
-    ] + condition_part + [
-        '  },',
-        '',
-    ])
+  for _, dep in sorted(deps.iteritems()):
+    s.extend(dep.ToLines())
   s.extend(['}', ''])
   return s
 

gclient_eval.py

@@ -21,6 +21,22 @@ _GCLIENT_DEPS_SCHEMA = {
             # Optional condition string. The dep will only be processed
             # if the condition evaluates to True.
             schema.Optional('condition'): basestring,
+
+            schema.Optional('dep_type', default='git'): basestring,
+        },
+        # CIPD package.
+        {
+            'packages': [
+                {
+                    'package': basestring,
+
+                    'version': basestring,
+                }
+            ],
+
+            schema.Optional('condition'): basestring,
+
+            schema.Optional('dep_type', default='cipd'): basestring,
         },
     ),
 }
@@ -217,7 +233,7 @@ def Exec(content, global_scope, local_scope, filename='<unknown>'):
             filename,
             getattr(node_or_string, 'lineno', '<unknown>')))
 
-  _GCLIENT_SCHEMA.validate(local_scope)
+  return _GCLIENT_SCHEMA.validate(local_scope)
 
 
 def EvaluateCondition(condition, variables, referenced_variables=None):

gclient_scm.py

@@ -6,13 +6,17 @@
 
 from __future__ import print_function
 
+import collections
+import contextlib
 import errno
+import json
 import logging
 import os
 import posixpath
 import re
 import sys
 import tempfile
+import threading
 import traceback
 import urlparse
 
@@ -81,37 +85,6 @@ class GitDiffFilterer(DiffFiltererWrapper):
     return re.sub("[a|b]/" + self._current_file, self._replacement_file, line)
 
 
-### SCM abstraction layer
-
-# Factory Method for SCM wrapper creation
-
-def GetScmName(url):
-  if not url:
-    return None
-  url, _ = gclient_utils.SplitUrlRevision(url)
-  if url.endswith('.git'):
-    return 'git'
-  protocol = url.split('://')[0]
-  if protocol in (
-      'file', 'git', 'git+http', 'git+https', 'http', 'https', 'ssh', 'sso'):
-    return 'git'
-  return None
-
-
-def CreateSCM(url, root_dir=None, relpath=None, out_fh=None, out_cb=None):
-  SCM_MAP = {
-    'git' : GitWrapper,
-  }
-
-  scm_name = GetScmName(url)
-  if not scm_name in SCM_MAP:
-    raise gclient_utils.Error('No SCM found for url %s' % url)
-  scm_class = SCM_MAP[scm_name]
-  if not scm_class.BinaryExists():
-    raise gclient_utils.Error('%s command not found' % scm_name)
-  return scm_class(url, root_dir, relpath, out_fh, out_cb)
-
-
 # SCMWrapper base class
 
 class SCMWrapper(object):
@@ -238,11 +211,11 @@ class GitWrapper(SCMWrapper):
 
   cache_dir = None
 
-  def __init__(self, url=None, *args):
+  def __init__(self, url=None, *args, **kwargs):
     """Removes 'git+' fake prefix from git URL."""
     if url.startswith('git+http://') or url.startswith('git+https://'):
       url = url[4:]
-    SCMWrapper.__init__(self, url, *args)
+    SCMWrapper.__init__(self, url, *args, **kwargs)
     filter_kwargs = { 'time_throttle': 1, 'out_fh': self.out_fh }
     if self.out_cb:
       filter_kwargs['predicate'] = self.out_cb
@@ -1230,3 +1203,217 @@ class GitWrapper(SCMWrapper):
       gclient_utils.CheckCallAndFilterAndHeader(cmd, env=env, **kwargs)
     else:
       gclient_utils.CheckCallAndFilter(cmd, env=env, **kwargs)
+
+
+class CipdPackage(object):
+  """A representation of a single CIPD package."""
+
+  def __init__(self, name, version, authority_for_root, authority_for_subdir):
+    self._authority_for_root = authority_for_root
+    self._authority_for_subdir = authority_for_subdir
+    self._name = name
+    self._version = version
+
+  @property
+  def authority_for_root(self):
+    """Whether this package has authority to act on behalf of its root.
+
+    Some operations should only be performed once per cipd root. A package
+    that has authority for its cipd root is the only package that should
+    perform such operations.
+
+    Returns:
+      bool; whether this package has root authority.
+    """
+    return self._authority_for_root
+
+  @property
+  def authority_for_subdir(self):
+    """Whether this package has authority to act on behalf of its subdir.
+
+    Some operations should only be performed once per subdirectory. A package
+    that has authority for its subdirectory is the only package that should
+    perform such operations.
+
+    Returns:
+      bool; whether this package has subdir authority.
+    """
+    return self._authority_for_subdir
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def version(self):
+    return self._version
+
+
+class CipdRoot(object):
+  """A representation of a single CIPD root."""
+  def __init__(self, root_dir, service_url):
+    self._all_packages = set()
+    self._mutator_lock = threading.Lock()
+    self._packages_by_subdir = collections.defaultdict(list)
+    self._root_dir = root_dir
+    self._service_url = service_url
+
+  def add_package(self, subdir, package, version):
+    """Adds a package to this CIPD root.
+
+    As far as clients are concerned, this grants both root and subdir authority
+    to packages arbitrarily. (The implementation grants root authority to the
+    first package added and subdir authority to the first package added for that
+    subdir, but clients should not depend on or expect that behavior.)
+
+    Args:
+      subdir: str; relative path to where the package should be installed from
+        the cipd root directory.
+      package: str; the cipd package name.
+      version: str; the cipd package version.
+    Returns:
+      CipdPackage; the package that was created and added to this root.
+    """
+    with self._mutator_lock:
+      cipd_package = CipdPackage(
+          package, version,
+          not self._packages_by_subdir,
+          not self._packages_by_subdir[subdir])
+      self._all_packages.add(cipd_package)
+      self._packages_by_subdir[subdir].append(cipd_package)
+      return cipd_package
+
+  def packages(self, subdir):
+    """Get the list of configured packages for the given subdir."""
+    return list(self._packages_by_subdir[subdir])
+
+  def clobber(self):
+    """Remove the .cipd directory.
+
+    This is useful for forcing ensure to redownload and reinitialize all
+    packages.
+    """
+    with self._mutator_lock:
+      cipd_cache_dir = os.path.join(self._cipd_root, '.cipd')
+      try:
+        gclient_utils.rmtree(os.path.join(cipd_cache_dir))
+      except OSError:
+        if os.path.exists(cipd_cache_dir):
+          raise
+
+  @contextlib.contextmanager
+  def _create_ensure_file(self):
+    try:
+      ensure_file = None
+      with tempfile.NamedTemporaryFile(
+          suffix='.ensure', delete=False) as ensure_file:
+        for subdir, packages in sorted(self._packages_by_subdir.iteritems()):
+          ensure_file.write('@Subdir %s\n' % subdir)
+          for package in packages:
+            ensure_file.write('%s %s\n' % (package.name, package.version))
+          ensure_file.write('\n')
+      yield ensure_file.name
+    finally:
+      if ensure_file is not None and os.path.exists(ensure_file.name):
+        os.remove(ensure_file.name)
+
+  def ensure(self):
+    """Run `cipd ensure`."""
+    with self._mutator_lock:
+      with self._create_ensure_file() as ensure_file:
+        cmd = [
+            'cipd', 'ensure',
+            '-log-level', 'error',
+            '-root', self.root_dir,
+            '-ensure-file', ensure_file,
+        ]
+        gclient_utils.CheckCallAndFilterAndHeader(cmd)
+
+  def created_package(self, package):
+    """Checks whether this root created the given package.
+
+    Args:
+      package: CipdPackage; the package to check.
+    Returns:
+      bool; whether this root created the given package.
+    """
+    return package in self._all_packages
+
+  @property
+  def root_dir(self):
+    return self._root_dir
+
+  @property
+  def service_url(self):
+    return self._service_url
+
+
+class CipdWrapper(SCMWrapper):
+  """Wrapper for CIPD.
+
+  Currently only supports chrome-infra-packages.appspot.com.
+  """
+
+  def __init__(self, url=None, root_dir=None, relpath=None, out_fh=None,
+               out_cb=None, root=None, package=None):
+    super(CipdWrapper, self).__init__(
+        url=url, root_dir=root_dir, relpath=relpath, out_fh=out_fh,
+        out_cb=out_cb)
+    assert root.created_package(package)
+    self._package = package
+    self._root = root
+
+  #override
+  def GetCacheMirror(self):
+    return None
+
+  #override
+  def GetActualRemoteURL(self, options):
+    return self._root.service_url
+
+  #override
+  def DoesRemoteURLMatch(self, options):
+    del options
+    return True
+
+  def revert(self, options, args, file_list):
+    """Deletes .cipd and reruns ensure."""
+    if self._package.authority_for_root:
+      self._root.clobber()
+      self._root.ensure()
+
+  def diff(self, options, args, file_list):
+    """CIPD has no notion of diffing."""
+    pass
+
+  def pack(self, options, args, file_list):
+    """CIPD has no notion of diffing."""
+    pass
+
+  def revinfo(self, options, args, file_list):
+    """Grab the instance ID."""
+    try:
+      tmpdir = tempfile.mkdtemp()
+      describe_json_path = os.path.join(tmpdir, 'describe.json')
+      cmd = [
+          'cipd', 'describe',
+          self._package.name,
+          '-log-level', 'error',
+          '-version', self._package.version,
+          '-json-output', describe_json_path
+      ]
+      gclient_utils.CheckCallAndFilter(
+          cmd, filter_fn=lambda _line: None, print_stdout=False)
+      with open(describe_json_path) as f:
+        describe_json = json.load(f)
+      return describe_json.get('result', {}).get('pin', {}).get('instance_id')
+    finally:
+      gclient_utils.rmtree(tmpdir)
+
+  def status(self, options, args, file_list):
+    pass
+
+  def update(self, options, args, file_list):
+    """Runs ensure."""
+    if self._package.authority_for_root:
+      self._root.ensure()

tests/gclient_scm_test.py

@@ -11,6 +11,7 @@
 from shutil import rmtree
 from subprocess import Popen, PIPE, STDOUT
 
+import json
 import logging
 import os
 import re
@@ -65,7 +66,7 @@ class BaseTestCase(GCBaseTestCase, SuperMoxTestBase):
     self.mox.StubOutWithMock(gclient_scm.gclient_utils, 'rmtree')
     self.mox.StubOutWithMock(subprocess2, 'communicate')
     self.mox.StubOutWithMock(subprocess2, 'Popen')
-    self._scm_wrapper = gclient_scm.CreateSCM
+    self._scm_wrapper = gclient_scm.GitWrapper
     self._original_GitBinaryExists = gclient_scm.GitWrapper.BinaryExists
     gclient_scm.GitWrapper.BinaryExists = staticmethod(lambda : True)
     # Absolute path of the fake checkout directory.
@@ -256,8 +257,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
       return
     options = self.Options()
     file_path = join(self.base_path, 'a')
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, None, file_list)
     gclient_scm.os.remove(file_path)
@@ -273,8 +274,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, None, file_list)
     file_list = []
@@ -288,8 +289,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, None, file_list)
     file_path = join(self.base_path, 'a')
@@ -308,8 +309,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, None, file_list)
     file_path = join(self.base_path, 'c')
@@ -334,8 +335,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     options = self.Options()
     file_path = join(self.base_path, 'a')
     open(file_path, 'a').writelines('touched\n')
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.status(options, self.args, file_list)
     self.assertEquals(file_list, [file_path])
@@ -353,8 +354,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
       file_path = join(self.base_path, f)
       open(file_path, 'a').writelines('touched\n')
       expected_file_list.extend([file_path])
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.status(options, self.args, file_list)
     expected_file_list = [join(self.base_path, x) for x in ['a', 'b']]
@@ -369,8 +370,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
       return
     options = self.Options()
     expected_file_list = [join(self.base_path, x) for x in ['a', 'b']]
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, (), file_list)
     self.assertEquals(file_list, expected_file_list)
@@ -383,8 +384,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
       return
     options = self.Options()
     options.merge = True
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     scm._Run(['checkout', '-q', 'feature'], options)
     rev = scm.revinfo(options, (), None)
     file_list = []
@@ -404,8 +405,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     scm._Run(['checkout', '-q', 'feature'], options)
     file_list = []
     # Fake a 'y' key press.
@@ -436,8 +437,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     file_path = join(self.base_path, 'file')
     open(file_path, 'w').writelines('new\n')
 
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, (), file_list)
     self.assert_(gclient_scm.os.path.isdir(dir_path))
@@ -458,8 +459,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     file_path = join(self.base_path, 'file')
     open(file_path, 'w').writelines('new\n')
 
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     scm.update(options, (), file_list)
     self.assert_(not gclient_scm.os.path.isdir(dir_path))
@@ -470,8 +471,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_path = join(self.base_path, 'b')
     open(file_path, 'w').writelines('conflict\n')
     try:
@@ -491,8 +492,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_path = join(self.base_path, '.git', 'index.lock')
     with open(file_path, 'w'):
       pass
@@ -505,8 +506,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
       return
     options = self.Options()
     options.break_repo_locks = True
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_path = join(self.base_path, '.git', 'index.lock')
     with open(file_path, 'w'):
       pass
@@ -520,8 +521,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_path = join(self.base_path, 'b')
     open(file_path, 'w').writelines('conflict\n')
     scm._Run(['commit', '-am', 'test'], options)
@@ -542,8 +543,8 @@ class ManagedGitWrapperTestCase(BaseGitWrapperTestCase):
     if not self.enabled:
       return
     options = self.Options()
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     rev_info = scm.revinfo(options, (), None)
     self.assertEquals(rev_info, '069c602044c5388d2d15c3f875b057c852003458')
 
@@ -605,8 +606,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
 
     self.mox.ReplayAll()
 
-    git_scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                    relpath=self.relpath)
+    git_scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                     self.relpath)
     # A [fake] git sha1 with a git repo should work (this is in the case that
     # the LKGR gets flipped to git sha1's some day).
     self.assertEquals(git_scm.GetUsableRev(self.fake_hash_1, options),
@@ -641,8 +642,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
         ).AndReturn('')
 
     self.mox.ReplayAll()
-    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
-                            relpath=self.relpath)
+    scm = self._scm_wrapper(self.url, self.root_dir,
+                            self.relpath)
     scm.update(options, None, [])
     self.checkstdout('\n')
 
@@ -678,8 +679,8 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
         ).AndReturn('')
 
     self.mox.ReplayAll()
-    scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
-                            relpath=self.relpath)
+    scm = self._scm_wrapper(self.url, self.root_dir,
+                            self.relpath)
     scm.update(options, None, [])
     self.checkstdout('\n')
 
@@ -715,9 +716,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
     self.relpath = '.'
     self.base_path = join(self.root_dir, self.relpath)
 
-    scm = gclient_scm.CreateSCM(url=origin_root_dir,
-                                root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(origin_root_dir,
+                                 self.root_dir,
+                                 self.relpath)
 
     expected_file_list = [join(self.base_path, "a"),
                           join(self.base_path, "b")]
@@ -747,9 +748,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
     url_with_commit_ref = origin_root_dir +\
                           '@a7142dc9f0009350b96a11f372b6ea658592aa95'
 
-    scm = gclient_scm.CreateSCM(url=url_with_commit_ref,
-                                root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(url_with_commit_ref,
+                                 self.root_dir,
+                                 self.relpath)
 
     expected_file_list = [join(self.base_path, "a"),
                           join(self.base_path, "b")]
@@ -778,9 +779,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
     self.base_path = join(self.root_dir, self.relpath)
     url_with_branch_ref = origin_root_dir + '@feature'
 
-    scm = gclient_scm.CreateSCM(url=url_with_branch_ref,
-                                root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(url_with_branch_ref,
+                                 self.root_dir,
+                                 self.relpath)
 
     expected_file_list = [join(self.base_path, "a"),
                           join(self.base_path, "b"),
@@ -811,9 +812,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
     self.base_path = join(self.root_dir, self.relpath)
     url_with_branch_ref = origin_root_dir + '@refs/remotes/origin/feature'
 
-    scm = gclient_scm.CreateSCM(url=url_with_branch_ref,
-                                root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(url_with_branch_ref,
+                                 self.root_dir,
+                                 self.relpath)
 
     expected_file_list = [join(self.base_path, "a"),
                           join(self.base_path, "b"),
@@ -843,9 +844,9 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
     self.base_path = join(self.root_dir, self.relpath)
     url_with_branch_ref = origin_root_dir + '@refs/heads/feature'
 
-    scm = gclient_scm.CreateSCM(url=url_with_branch_ref,
-                                root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(url_with_branch_ref,
+                                 self.root_dir,
+                                 self.relpath)
 
     expected_file_list = [join(self.base_path, "a"),
                           join(self.base_path, "b"),
@@ -876,8 +877,8 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
       return
     options = self.Options()
     expected_file_list = []
-    scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
-                                relpath=self.relpath)
+    scm = gclient_scm.GitWrapper(self.url, self.root_dir,
+                                 self.relpath)
     file_list = []
     options.revision = 'unmanaged'
     scm.update(options, (), file_list)
@@ -887,6 +888,129 @@ class UnmanagedGitWrapperTestCase(BaseGitWrapperTestCase):
     self.checkstdout('________ unmanaged solution; skipping .\n')
 
 
+class CipdWrapperTestCase(BaseTestCase):
+
+  def setUp(self):
+    # Create this before setting up mocks.
+    self._cipd_root_dir = tempfile.mkdtemp()
+    self._workdir = tempfile.mkdtemp()
+    BaseTestCase.setUp(self)
+
+    self._cipd_instance_url = 'https://chrome-infra-packages.appspot.com'
+    self._cipd_root = gclient_scm.CipdRoot(
+        self._cipd_root_dir,
+        self._cipd_instance_url)
+    self._cipd_packages = [
+        self._cipd_root.add_package('f', 'foo_package', 'foo_version'),
+        self._cipd_root.add_package('b', 'bar_package', 'bar_version'),
+        self._cipd_root.add_package('b', 'baz_package', 'baz_version'),
+    ]
+    self.mox.StubOutWithMock(gclient_scm.CipdRoot, 'add_package')
+    self.mox.StubOutWithMock(gclient_scm.CipdRoot, 'clobber')
+    self.mox.StubOutWithMock(gclient_scm.CipdRoot, 'ensure')
+
+  def tearDown(self):
+    BaseTestCase.tearDown(self)
+    rmtree(self._cipd_root_dir)
+    rmtree(self._workdir)
+
+  def createScmWithPackageThatSatisfies(self, condition):
+    return gclient_scm.CipdWrapper(
+        url=self._cipd_instance_url,
+        root_dir=self._cipd_root_dir,
+        relpath='fake_relpath',
+        root=self._cipd_root,
+        package=self.getPackageThatSatisfies(condition))
+
+  def getPackageThatSatisfies(self, condition):
+    for p in self._cipd_packages:
+      if condition(p):
+        return p
+
+    self.fail('Unable to find a satisfactory package.')
+
+  def testSingleRootAuthority(self):
+    """Checks that exactly one package has root authority."""
+    self.assertEquals(1, len([p for p in self._cipd_packages
+                              if p.authority_for_root]))
+
+  def testRevert(self):
+    """Checks that revert w/ root authority clobbers and reruns ensure."""
+    scm = self.createScmWithPackageThatSatisfies(
+        lambda p: p.authority_for_root)
+
+    gclient_scm.CipdRoot.clobber()
+    gclient_scm.CipdRoot.ensure()
+
+    self.mox.ReplayAll()
+
+    scm.revert(None, (), [])
+
+  def testRevertWithoutAuthority(self):
+    """Checks that revert w/o root authority does nothing."""
+    scm = self.createScmWithPackageThatSatisfies(
+        lambda p: not p.authority_for_root)
+
+    self.mox.ReplayAll()
+
+    scm.revert(None, (), [])
+
+  def testRevinfo(self):
+    """Checks that revinfo uses the JSON from cipd describe."""
+    scm = self.createScmWithPackageThatSatisfies(lambda _: True)
+
+    expected_revinfo = '0123456789abcdef0123456789abcdef01234567'
+    json_contents = {
+        'result': {
+            'pin': {
+                'instance_id': expected_revinfo,
+            }
+        }
+    }
+    describe_json_path = join(self._workdir, 'describe.json')
+    with open(describe_json_path, 'w') as describe_json:
+      json.dump(json_contents, describe_json)
+
+    cmd = [
+        'cipd', 'describe', 'foo_package',
+        '-log-level', 'error',
+        '-version', 'foo_version',
+        '-json-output', describe_json_path,
+    ]
+
+    self.mox.StubOutWithMock(tempfile, 'mkdtemp')
+
+    tempfile.mkdtemp().AndReturn(self._workdir)
+    gclient_scm.gclient_utils.CheckCallAndFilter(
+        cmd, filter_fn=mox.IgnoreArg(), print_stdout=False)
+    gclient_scm.gclient_utils.rmtree(self._workdir)
+
+    self.mox.ReplayAll()
+
+    revinfo = scm.revinfo(None, (), [])
+    self.assertEquals(revinfo, expected_revinfo)
+
+  def testUpdate(self):
+    """Checks that update w/ root authority runs ensure."""
+    scm = self.createScmWithPackageThatSatisfies(
+        lambda p: p.authority_for_root)
+
+    gclient_scm.CipdRoot.ensure()
+
+    self.mox.ReplayAll()
+
+    scm.update(None, (), [])
+
+  def testUpdateWithoutAuthority(self):
+    """Checks that update w/o root authority does nothing."""
+    scm = self.createScmWithPackageThatSatisfies(
+        lambda p: not p.authority_for_root)
+
+    self.mox.ReplayAll()
+
+    scm.update(None, (), [])
+
+
 if __name__ == '__main__':
   level = logging.DEBUG if '-v' in sys.argv else logging.FATAL
   logging.basicConfig(

tests/gclient_test.py

@@ -60,15 +60,15 @@ class GclientTest(trial_dir.TestCase):
     self.previous_dir = os.getcwd()
     os.chdir(self.root_dir)
     # Manual mocks.
-    self._old_createscm = gclient.gclient_scm.CreateSCM
-    gclient.gclient_scm.CreateSCM = self._createscm
+    self._old_createscm = gclient.Dependency.CreateSCM
+    gclient.Dependency.CreateSCM = self._createscm
     self._old_sys_stdout = sys.stdout
     sys.stdout = gclient.gclient_utils.MakeFileAutoFlush(sys.stdout)
     sys.stdout = gclient.gclient_utils.MakeFileAnnotated(sys.stdout)
 
   def tearDown(self):
     self.assertEquals([], self._get_processed())
-    gclient.gclient_scm.CreateSCM = self._old_createscm
+    gclient.Dependency.CreateSCM = self._old_createscm
     sys.stdout = self._old_sys_stdout
     os.chdir(self.previous_dir)
     super(GclientTest, self).tearDown()