Add -j option to gclient to run parallel updates

On a chromeos checkout, -j 10 brings down null sync time from 2 minutes to 16 seconds
Currently -j may break some assumptions about ordering of commands (which may be fine for some clients, but not all), so it's not on by default.

Review URL: http://codereview.chromium.org/1640001

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@44869 0039d316-1c4b-4281-b951-d872f2087c98

gclient.py

@@ -58,6 +58,7 @@ Hooks
 __author__ = "darinf@gmail.com (Darin Fisher)"
 __version__ = "0.3.4"
 
+import copy
 import errno
 import logging
 import optparse
@@ -698,6 +699,17 @@ class GClient(object):
       if matching_file_list:
         self._RunHookAction(hook_dict, matching_file_list)
 
+  def GetSCMCommandClosure(self, path, url, revision, command, args, file_list):
+    """Gets a closure that runs a SCM command on a particular dependency."""
+    def _Closure():
+      logging.debug("Running %s in %s to %s %s" % (command, path, url,
+                                                   revision))
+      options = copy.copy(self._options)
+      options.revision = revision
+      scm = gclient_scm.CreateSCM(url, self._root_dir, path)
+      scm.RunCommand(command, options, args, file_list)
+    return _Closure
+
   def RunOnDeps(self, command, args):
     """Runs a command on each dependency in a client and its dependencies.
 
@@ -738,84 +750,125 @@ class GClient(object):
 
     entries = {}
     entries_deps_content = {}
-    file_list = []
-    # Run on the base solutions first.
-    for solution in solutions:
-      name = solution["name"]
-      deps_file = solution.get("deps_file", self._options.deps_file)
-      if '/' in deps_file or '\\' in deps_file:
-        raise gclient_utils.Error('deps_file name must not be a path, just a '
-                                  'filename.')
-      if name in entries:
-        raise gclient_utils.Error("solution %s specified more than once" % name)
-      url = solution["url"]
-      entries[name] = url
-      if run_scm and url:
-        self._options.revision = revision_overrides.get(name)
-        scm = gclient_scm.CreateSCM(url, self._root_dir, name)
-        scm.RunCommand(command, self._options, args, file_list)
-        file_list = [os.path.join(name, f.strip()) for f in file_list]
-        self._options.revision = None
-      try:
-        deps_content = gclient_utils.FileRead(
-            os.path.join(self._root_dir, name, deps_file))
-      except IOError, e:
-        if e.errno != errno.ENOENT:
-          raise
-        deps_content = ""
-      entries_deps_content[name] = deps_content
 
-    # Process the dependencies next (sort alphanumerically to ensure that
-    # containing directories get populated first and for readability)
-    deps = self._ParseAllDeps(entries, entries_deps_content)
-    deps_to_process = deps.keys()
-    deps_to_process.sort()
+    # To avoid threading issues, all file lists get constructed separately then
+    # gathered in a flattened list at the end.
+    file_list_list = []
+    file_list_dict = {}
 
-    # First pass for direct dependencies.
-    if command == 'update' and not self._options.verbose:
-      pm = Progress('Syncing projects', len(deps_to_process))
-    for d in deps_to_process:
+    thread_pool = gclient_utils.ThreadPool(self._options.jobs)
+    thread_pool.Start()
+
+    try:
+      # Run on the base solutions first.
+      for solution in solutions:
+        name = solution["name"]
+        deps_file = solution.get("deps_file", self._options.deps_file)
+        if '/' in deps_file or '\\' in deps_file:
+          raise gclient_utils.Error('deps_file name must not be a path, just a '
+                                    'filename.')
+        if name in entries:
+          raise gclient_utils.Error(
+              "solution %s specified more than once" % name)
+        url = solution["url"]
+        entries[name] = url
+        if run_scm and url:
+          revision = revision_overrides.get(name)
+          file_list = []
+          file_list_dict[name] = file_list
+          thread_pool.AddJob(self.GetSCMCommandClosure(
+              name, url, revision, command, args, file_list))
+
+      thread_pool.WaitJobs()
+
+      for solution in solutions:
+        name = solution["name"]
+        deps_file = solution.get("deps_file", self._options.deps_file)
+        try:
+          deps_content = gclient_utils.FileRead(
+              os.path.join(self._root_dir, name, deps_file))
+        except IOError, e:
+          if e.errno != errno.ENOENT:
+            raise
+          deps_content = ""
+        entries_deps_content[name] = deps_content
+        try:
+          file_list_list.append([os.path.join(name, f.strip())
+                                 for f in file_list_dict[name]])
+        except KeyError:
+          # We may not have added the file list to the dict, see tests above.
+          # Instead of duplicating the tests, it's less fragile to just ignore
+          # the exception.
+          pass
+
+      # Process the dependencies next (sort alphanumerically to ensure that
+      # containing directories get populated first and for readability)
+      # TODO(piman): when using multiple threads, the ordering is not ensured.
+      # In many cases (e.g. updates to an existing checkout where DEPS don't
+      # move between directories), it'll still be correct but for completeness
+      # this should be fixed.
+      deps = self._ParseAllDeps(entries, entries_deps_content)
+      deps_to_process = deps.keys()
+      deps_to_process.sort()
+
+      # First pass for direct dependencies.
       if command == 'update' and not self._options.verbose:
-        pm.update()
-      if type(deps[d]) == str:
-        url = deps[d]
-        entries[d] = url
-        if run_scm:
-          self._options.revision = revision_overrides.get(d)
-          scm = gclient_scm.CreateSCM(url, self._root_dir, d)
-          scm.RunCommand(command, self._options, args, file_list)
-          self._options.revision = None
-      elif isinstance(deps[d], self.FileImpl):
-        file = deps[d]
-        self._options.revision = file.GetRevision()
-        if run_scm:
-          scm = gclient_scm.CreateSCM(file.GetPath(), self._root_dir, d)
-          scm.RunCommand("updatesingle", self._options,
-                         args + [file.GetFilename()], file_list)
+        pm = Progress('Syncing projects', len(deps_to_process))
+
+      for d in deps_to_process:
+        if command == 'update' and not self._options.verbose:
+          pm.update()
+        file_list = []
+        file_list_list.append(file_list)
+        if type(deps[d]) == str:
+          url = deps[d]
+          entries[d] = url
+          if run_scm:
+            revision = revision_overrides.get(d)
+            thread_pool.AddJob(self.GetSCMCommandClosure(d, url, revision,
+                                                         command, args,
+                                                         file_list))
+        elif isinstance(deps[d], self.FileImpl):
+          file = deps[d]
+          if run_scm:
+            revision = file.GetRevision()
+            thread_pool.AddJob(self.GetSCMCommandClosure(
+                d, url, revision, "updatesingle", args + [file.GetFilename()],
+                file_list))
+
+      thread_pool.WaitJobs()
           
-    if command == 'update' and not self._options.verbose:
-      pm.end()
+      if command == 'update' and not self._options.verbose:
+        pm.end()
+
+      # Second pass for inherited deps (via the From keyword)
+      for d in deps_to_process:
+        if isinstance(deps[d], self.FromImpl):
+          filename = os.path.join(self._root_dir,
+                                  deps[d].module_name,
+                                  self._options.deps_file)
+          content =  gclient_utils.FileRead(filename)
+          sub_deps = self._ParseSolutionDeps(deps[d].module_name, content, {},
+                                             False)
+          # Getting the URL from the sub_deps file can involve having to resolve
+          # a File() or having to resolve a relative URL.  To resolve relative
+          # URLs, we need to pass in the orignal sub deps URL.
+          sub_deps_base_url = deps[deps[d].module_name]
+          url = deps[d].GetUrl(d, sub_deps_base_url, self._root_dir, sub_deps)
+          entries[d] = url
+          if run_scm:
+            revision = revision_overrides.get(d)
+            file_list = []
+            file_list_list.append(file_list)
+            thread_pool.AddJob(self.GetSCMCommandClosure(d, url, revision,
+                                                         command, args,
+                                                         file_list))
 
-    # Second pass for inherited deps (via the From keyword)
-    for d in deps_to_process:
-      if isinstance(deps[d], self.FromImpl):
-        filename = os.path.join(self._root_dir,
-                                deps[d].module_name,
-                                self._options.deps_file)
-        content =  gclient_utils.FileRead(filename)
-        sub_deps = self._ParseSolutionDeps(deps[d].module_name, content, {},
-                                           False)
-        # Getting the URL from the sub_deps file can involve having to resolve
-        # a File() or having to resolve a relative URL.  To resolve relative
-        # URLs, we need to pass in the orignal sub deps URL.
-        sub_deps_base_url = deps[deps[d].module_name]
-        url = deps[d].GetUrl(d, sub_deps_base_url, self._root_dir, sub_deps)
-        entries[d] = url
-        if run_scm:
-          self._options.revision = revision_overrides.get(d)
-          scm = gclient_scm.CreateSCM(url, self._root_dir, d)
-          scm.RunCommand(command, self._options, args, file_list)
-          self._options.revision = None
+      thread_pool.WaitJobs()
+    finally:
+      thread_pool.Stop()
+
+    file_list = sum(file_list_list, [])
 
     # Convert all absolute paths to relative.
     for i in range(len(file_list)):
@@ -1284,6 +1337,9 @@ def Main(argv):
   option_parser.add_option("", "--gclientfile", default=None,
                            metavar="FILENAME",
                            help=("specify an alternate .gclient file"))
+  option_parser.add_option("-j", "--jobs", default=1, type="int",
+                           help=("specify how many SCM commands can run in "
+                                 "parallel"))
 
   if len(argv) < 2:
     # Users don't need to be told to use the 'help' command.

gclient_utils.py

@@ -17,11 +17,14 @@
 import errno
 import logging
 import os
+import Queue
 import re
 import stat
 import subprocess
 import sys
 import time
+import threading
+import traceback
 import xml.dom.minidom
 import xml.parsers.expat
 
@@ -353,3 +356,80 @@ def GetGClientRootAndEntries(path=None):
   execfile(config_path, env)
   config_dir = os.path.dirname(config_path)
   return config_dir, env['entries']
+
+
+class ThreadPool:
+  """A thread pool class that lets one schedule jobs on many worker threads."""
+
+  def __init__(self, threads=1):
+    self._threads = threads
+    self._queue = Queue.Queue()
+    self._workers = []
+
+  class Worker(threading.Thread):
+    """Internal worker class that executes jobs from the ThreadPool queue."""
+
+    def __init__(self, pool):
+      threading.Thread.__init__(self)
+      self._pool = pool
+      self._done = False
+      self.exceptions = []
+
+    def Done(self):
+      """Terminates the worker threads."""
+      self._done = True
+
+    def run(self):
+      """Executes jobs from the pool's queue."""
+      while not self._done:
+        f = self._pool._queue.get()
+        try:
+          f(self)
+        except Exception, e:
+          # Catch all exceptions, otherwise we can't join the thread. Print the
+          # backtrace now, but keep the exception so that we can raise it on the
+          # main thread.
+          type, value, tb = sys.exc_info()
+          traceback.print_exception(type, value, tb)
+          self.exceptions.append(e)
+        finally:
+          self._pool._queue.task_done()
+
+
+  def Start(self):
+    """Starts the thread pool. Spawns worker threads."""
+    assert not self._workers
+    for i in xrange(0, self._threads):
+      worker = self.Worker(self)
+      self._workers.append(worker)
+      worker.start()
+
+  def Stop(self):
+    """Stops the thread pool. Joins all worker threads."""
+    assert self._workers
+    for i in xrange(0, len(self._workers)):
+      wrapped = lambda thread: thread.Done()
+      self._queue.put(wrapped)
+    self._queue.join()
+    for worker in self._workers:
+      worker.join()
+    try:
+      for worker in self._workers:
+        for e in worker.exceptions:
+          # If we collected exceptions, raise them now.
+          raise e
+    finally:
+      self._workers = []
+
+  def AddJob(self, function):
+    """Adds a job to the queue.
+    
+    A job is a simple closure, that will get executed on one of the worker
+    threads."""
+    wrapped = lambda worker: function()
+    self._queue.put(wrapped)
+
+  def WaitJobs(self):
+    """Waits for all jobs to be completed."""
+    assert self._workers
+    self._queue.join()

tests/gclient_test.py

@@ -55,7 +55,8 @@ class GclientTestCase(GClientBaseTestCase):
     def __init__(self, test_case, verbose=False, spec=None,
                  config_filename='a_file_name',
                  entries_filename='a_entry_file_name',
-                 deps_file='a_deps_file_name', force=False, nohooks=False):
+                 deps_file='a_deps_file_name', force=False, nohooks=False,
+                 jobs=1):
       self.verbose = verbose
       self.spec = spec
       self.name = None
@@ -64,6 +65,8 @@ class GclientTestCase(GClientBaseTestCase):
       self.deps_file = deps_file
       self.force = force
       self.nohooks = nohooks
+      self.jobs = jobs
+      self.revision = None
       self.revisions = []
       self.manually_grab_svn_rev = True
       self.deps_os = None
@@ -296,12 +299,27 @@ class TestDoRevert(GenericCommandTestCase):
     self.BadClient(gclient.DoRevert)
 
 
+def CompareOptions(options):
+  def _Compare(other):
+    options_keys = [i for i in dir(options) if not i.startswith('_')]
+    other_keys = [i for i in dir(other) if not i.startswith('_')]
+    if set(options_keys) != set(other_keys):
+      return False
+    try:
+      for key in options_keys:
+        if getattr(options, key) != getattr(other, key):
+          return False
+      return True
+    except AttributeError:
+      return False
+  return _Compare
+
 class GClientClassTestCase(GclientTestCase):
   def testDir(self):
     members = [
       'ConfigContent', 'FileImpl', 'FromImpl', 'GetVar', 'LoadCurrentConfig',
       'RunOnDeps', 'SaveConfig', 'SetConfig', 'SetDefaultConfig',
-      'supported_commands', 'PrintRevInfo',
+      'supported_commands', 'PrintRevInfo', 'GetSCMCommandClosure',
     ]
 
     # If you add a member, be sure to add the relevant test!
@@ -384,7 +402,8 @@ class GClientClassTestCase(GclientTestCase):
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, solution_name
         ).AndReturn(scm_wrapper_sol)
     # Then an update will be performed.
-    scm_wrapper_sol.RunCommand('update', options, self.args, [])
+    scm_wrapper_sol.RunCommand('update', mox.Func(CompareOptions(options)),
+                               self.args, [])
     # Then an attempt will be made to read its DEPS file.
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, solution_name, options.deps_file)
@@ -444,7 +463,8 @@ class GClientClassTestCase(GclientTestCase):
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, solution_name
         ).AndReturn(scm_wrapper_sol)
     # Then an update will be performed.
-    scm_wrapper_sol.RunCommand('update', options, self.args, [])
+    scm_wrapper_sol.RunCommand('update', mox.Func(CompareOptions(options)),
+                               self.args, [])
     # Then an attempt will be made to read its DEPS file.
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, solution_name, options.deps_file)
@@ -458,7 +478,8 @@ class GClientClassTestCase(GclientTestCase):
         self.root_dir,
         gclient.os.path.join(solution_name, "src", "t")
         ).AndReturn(scm_wrapper_t)
-    scm_wrapper_t.RunCommand('update', options, self.args, [])
+    scm_wrapper_t.RunCommand('update', mox.Func(CompareOptions(options)),
+                             self.args, [])
 
     # After everything is done, an attempt is made to write an entries
     # file.
@@ -521,7 +542,8 @@ class GClientClassTestCase(GclientTestCase):
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, solution_name
         ).AndReturn(scm_wrapper_sol)
     # Then an update will be performed.
-    scm_wrapper_sol.RunCommand('update', options, self.args, [])
+    scm_wrapper_sol.RunCommand('update', mox.Func(CompareOptions(options)),
+                               self.args, [])
     # Then an attempt will be made to read its DEPS file.
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(checkout_path, options.deps_file)
@@ -539,8 +561,10 @@ class GClientClassTestCase(GclientTestCase):
                                   self.root_dir,
                                   "src/t").AndReturn(scm_wrapper_t)
 
-    scm_wrapper_n.RunCommand('update', options, self.args, [])
-    scm_wrapper_t.RunCommand('update', options, self.args, [])
+    scm_wrapper_n.RunCommand('update', mox.Func(CompareOptions(options)),
+                             self.args, [])
+    scm_wrapper_t.RunCommand('update', mox.Func(CompareOptions(options)),
+                             self.args, [])
 
     # NOTE: the dep src/b should not create an scm at all.
 
@@ -615,7 +639,8 @@ class GClientClassTestCase(GclientTestCase):
         gclient.os.path.join(self.root_dir, name_a, options.deps_file)
         ).AndReturn(deps_a)
     # Then an update will be performed.
-    scm_wrapper_a.RunCommand('update', options, self.args, [])
+    scm_wrapper_a.RunCommand('update', mox.Func(CompareOptions(options)),
+                             self.args, [])
 
     # An scm will be requested for the second solution.
     gclient.gclient_scm.CreateSCM(url_b, self.root_dir, name_b).AndReturn(
@@ -625,13 +650,15 @@ class GClientClassTestCase(GclientTestCase):
         gclient.os.path.join(self.root_dir, name_b, options.deps_file)
         ).AndReturn(deps_b)
     # Then an update will be performed.
-    scm_wrapper_b.RunCommand('update', options, self.args, [])
+    scm_wrapper_b.RunCommand('update', mox.Func(CompareOptions(options)),
+                             self.args, [])
 
     # Finally, an scm is requested for the shared dep.
     gclient.gclient_scm.CreateSCM('http://svn.t/trunk', self.root_dir, 'src/t'
         ).AndReturn(scm_wrapper_dep)
     # And an update is run on it.
-    scm_wrapper_dep.RunCommand('update', options, self.args, [])
+    scm_wrapper_dep.RunCommand('update', mox.Func(CompareOptions(options)),
+                               self.args, [])
 
     # After everything is done, an attempt is made to write an entries file.
     gclient.gclient_utils.FileWrite(
@@ -666,7 +693,8 @@ class GClientClassTestCase(GclientTestCase):
         ).AndReturn(False)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, name, options.deps_file)
         ).AndReturn("Boo = 'a'")
@@ -886,7 +914,8 @@ deps = {
         ).AndReturn(False)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, None
                                   ).AndReturn(scm_wrapper_src)
@@ -896,7 +925,8 @@ deps = {
     gclient.gclient_scm.CreateSCM(
         webkit_path, self.root_dir, 'foo/third_party/WebKit'
         ).AndReturn(gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     self.mox.ReplayAll()
     client = self._gclient_gclient(self.root_dir, options)
@@ -950,7 +980,8 @@ deps = {
         ).AndReturn(False)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir,
                        None).AndReturn(scm_wrapper_src)
@@ -959,7 +990,8 @@ deps = {
 
     gclient.gclient_scm.CreateSCM(webkit_path, self.root_dir,
         'foo/third_party/WebKit').AndReturn(gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     self.mox.ReplayAll()
     client = self._gclient_gclient(self.root_dir, options)
@@ -986,7 +1018,8 @@ deps = {
         ).AndReturn(deps_content)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     self.mox.ReplayAll()
     client = self._gclient_gclient(self.root_dir, options)
@@ -1049,7 +1082,8 @@ deps = {
         ).AndReturn(False)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, name, options.deps_file)
         ).AndReturn(deps_content)
@@ -1057,7 +1091,8 @@ deps = {
     # base gets updated.
     gclient.gclient_scm.CreateSCM(base_url, self.root_dir, 'base').AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, 'base', options.deps_file)
         ).AndReturn(base_deps_content)
@@ -1065,7 +1100,8 @@ deps = {
     # main gets updated.
     gclient.gclient_scm.CreateSCM(main_url, self.root_dir, 'main').AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     # Process is done and will write an .gclient_entries.    
     gclient.os.path.exists(
@@ -1112,7 +1148,8 @@ deps = {
         ).AndReturn(False)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, name, options.deps_file)
         ).AndReturn(deps_content)
@@ -1120,7 +1157,8 @@ deps = {
     # base gets updated.
     gclient.gclient_scm.CreateSCM(base_url, self.root_dir, 'base').AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, 'base', options.deps_file)
         ).AndReturn(base_deps_content)
@@ -1128,7 +1166,8 @@ deps = {
     # main gets updated.
     gclient.gclient_scm.CreateSCM(main_url, self.root_dir, 'main').AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     # Process is done and will write an .gclient_entries.    
     gclient.os.path.exists(
@@ -1175,7 +1214,8 @@ deps = {
         ).AndReturn(False)
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, name, options.deps_file)
         ).AndReturn(deps_content)
@@ -1183,7 +1223,8 @@ deps = {
     # base gets updated.
     gclient.gclient_scm.CreateSCM(base_url, self.root_dir, 'base').AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, 'base', options.deps_file)
         ).AndReturn(base_deps_content)
@@ -1195,7 +1236,8 @@ deps = {
         ).AndReturn('svn://base' + main_url)
     gclient.gclient_scm.CreateSCM('svn://base' + main_url, self.root_dir,
         'main').AndReturn(gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
 
     # Process is done and will write an .gclient_entries.    
     gclient.os.path.exists(
@@ -1227,7 +1269,8 @@ deps = {
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
         gclient.gclient_scm.CreateSCM)
     options = self.Options()
-    gclient.gclient_scm.CreateSCM.RunCommand('update', options, self.args, [])
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        'update', mox.Func(CompareOptions(options)), self.args, [])
     gclient.gclient_utils.FileRead(
         gclient.os.path.join(self.root_dir, name, options.deps_file)
         ).AndReturn(deps_content)
@@ -1241,7 +1284,8 @@ deps = {
     # This is where gclient tries to do the initial checkout.
     gclient.gclient_scm.CreateSCM(self.url, self.root_dir, target).AndReturn(
         gclient.gclient_scm.CreateSCM)
-    gclient.gclient_scm.CreateSCM.RunCommand('updatesingle', options,
+    gclient.gclient_scm.CreateSCM.RunCommand('updatesingle',
+                                             mox.Func(CompareOptions(options)),
         self.args + ["DEPS"], [])
     gclient.gclient_utils.FileWrite(
         gclient.os.path.join(self.root_dir, options.entries_filename),
@@ -1257,6 +1301,25 @@ deps = {
     # implementation for Pulse plugin
     pass
 
+  def testGetSCMCommandClosure(self):
+    options = self.Options()
+    name = 'testGetSCMCommandClosure_name'
+    command = 'testGetSCMCommandClosure_command'
+    revision = '123'
+    file_list = []
+    called_options = self.Options()
+    called_options.revision = revision
+    gclient.gclient_scm.CreateSCM(self.url, self.root_dir, name).AndReturn(
+        gclient.gclient_scm.CreateSCM)
+    gclient.gclient_scm.CreateSCM.RunCommand(
+        command, mox.Func(CompareOptions(called_options)), self.args, file_list)
+
+    self.mox.ReplayAll()
+    client = self._gclient_gclient(self.root_dir, options)
+    closure = client.GetSCMCommandClosure(name, self.url, revision, command,
+                                          self.args, file_list)
+    closure()
+
   # No test for internal functions.
   def test_GetAllDeps(self):
     pass