From 064f6f4a9d2932ae699dbf4355bc63359e599026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Hajdan=2C=20Jr?= Date: Thu, 18 May 2017 22:17:55 +0200 Subject: [PATCH] gclient flatten: first pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is based on https://codereview.chromium.org/2474543002/ Bug: 661382 Change-Id: I191ec16e0ce69a782979ae7d59b108747429ab78 Reviewed-on: https://chromium-review.googlesource.com/505067 Commit-Queue: Paweł Hajdan Jr. Reviewed-by: Dirk Pranke Reviewed-by: Andrii Shyshkalov Reviewed-by: Michael Moss --- gclient.py | 184 +++++++++++++++++++++++++++++++++- testing_support/fake_repos.py | 29 +++++- tests/gclient_smoketest.py | 56 +++++++++++ 3 files changed, 264 insertions(+), 5 deletions(-) diff --git a/gclient.py b/gclient.py index e083437bf5..779ce51ac1 100755 --- a/gclient.py +++ b/gclient.py @@ -772,7 +772,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): # When running runhooks, there's no need to consult the SCM. # All known hooks are expected to run unconditionally regardless of working # copy state, so skip the SCM status check. - run_scm = command not in ('runhooks', 'recurse', 'validate', None) + run_scm = command not in ( + 'flatten', 'runhooks', 'recurse', 'validate', None) parsed_url = self.LateOverride(self.url) file_list = [] if not options.nohooks else None revision_override = revision_overrides.pop(self.name, None) @@ -1090,12 +1091,16 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): def __repr__(self): return '%s: %s' % (self.name, self.url) - def hierarchy(self): + def hierarchy(self, include_url=True): """Returns a human-readable hierarchical reference to a Dependency.""" - out = '%s(%s)' % (self.name, self.url) + def format_name(d): + if include_url: + return '%s(%s)' % (d.name, d.url) + return d.name + out = format_name(self) i = self.parent while i and i.name: - out = '%s(%s) -> %s' % (i.name, i.url, out) + out = '%s -> %s' % (format_name(i), out) i = i.parent return out @@ -1632,6 +1637,177 @@ def CMDfetch(parser, args): '--jobs=%d' % options.jobs, '--scm=git', 'git', 'fetch'] + args) +def CMDflatten(parser, args): + """Flattens the solutions into a single DEPS file.""" + parser.add_option('--output-deps', help='Path to the output DEPS file') + parser.add_option( + '--require-pinned-revisions', action='store_true', + help='Fail if any of the dependencies uses unpinned revision.') + options, args = parser.parse_args(args) + + options.nohooks = True + client = GClient.LoadCurrentConfig(options) + + # Only print progress if we're writing to a file. Otherwise, progress updates + # could obscure intended output. + code = client.RunOnDeps('flatten', args, progress=options.output_deps) + if code != 0: + return code + + deps = {} + hooks = [] + pre_deps_hooks = [] + unpinned_deps = {} + + for solution in client.dependencies: + _FlattenSolution(solution, deps, hooks, pre_deps_hooks, unpinned_deps) + + if options.require_pinned_revisions and unpinned_deps: + sys.stderr.write('The following dependencies are not pinned:\n') + sys.stderr.write('\n'.join(sorted(unpinned_deps))) + return 1 + + flattened_deps = '\n'.join( + _DepsToLines(deps) + + _HooksToLines('hooks', hooks) + + _HooksToLines('pre_deps_hooks', pre_deps_hooks) + + [''] # Ensure newline at end of file. + ) + + if options.output_deps: + with open(options.output_deps, 'w') as f: + f.write(flattened_deps) + else: + print(flattened_deps) + + return 0 + + +def _FlattenSolution(solution, deps, hooks, pre_deps_hooks, unpinned_deps): + """Visits a solution in order to flatten it (see CMDflatten). + + Arguments: + solution (Dependency): one of top-level solutions in .gclient + + Out-parameters: + deps (dict of name -> Dependency): will be filled with all Dependency + objects indexed by their name + hooks (list of (Dependency, hook)): will be filled with flattened hooks + pre_deps_hooks (list of (Dependency, hook)): will be filled with flattened + pre_deps_hooks + unpinned_deps (dict of name -> Dependency): will be filled with unpinned + deps + """ + logging.debug('_FlattenSolution(%r)', solution) + + _FlattenDep(solution, deps, hooks, pre_deps_hooks, unpinned_deps) + _FlattenRecurse(solution, deps, hooks, pre_deps_hooks, unpinned_deps) + + +def _FlattenDep(dep, deps, hooks, pre_deps_hooks, unpinned_deps): + """Visits a dependency in order to flatten it (see CMDflatten). + + Arguments: + dep (Dependency): dependency to process + + Out-parameters: + deps (dict): will be filled with flattened deps + hooks (list): will be filled with flattened hooks + pre_deps_hooks (list): will be filled with flattened pre_deps_hooks + unpinned_deps (dict): will be filled with unpinned deps + """ + logging.debug('_FlattenDep(%r)', dep) + + _AddDep(dep, deps, unpinned_deps) + + deps_by_name = dict((d.name, d) for d in dep.dependencies) + for recurse_dep_name in (dep.recursedeps or []): + _FlattenRecurse( + deps_by_name[recurse_dep_name], deps, hooks, pre_deps_hooks, + unpinned_deps) + + # TODO(phajdan.jr): also handle hooks_os. + hooks.extend([(dep, hook) for hook in dep.deps_hooks]) + pre_deps_hooks.extend( + [(dep, {'action': hook}) for hook in dep.pre_deps_hooks]) + + +def _FlattenRecurse(dep, deps, hooks, pre_deps_hooks, unpinned_deps): + """Helper for flatten that recurses into |dep|'s dependencies. + + Arguments: + dep (Dependency): dependency to process + + Out-parameters: + deps (dict): will be filled with flattened deps + hooks (list): will be filled with flattened hooks + pre_deps_hooks (list): will be filled with flattened pre_deps_hooks + unpinned_deps (dict): will be filled with unpinned deps + """ + logging.debug('_FlattenRecurse(%r)', dep) + + # TODO(phajdan.jr): also handle deps_os. + for dep in dep.dependencies: + _FlattenDep(dep, deps, hooks, pre_deps_hooks, unpinned_deps) + + +def _AddDep(dep, deps, unpinned_deps): + """Helper to add a dependency to flattened lists. + + Arguments: + dep (Dependency): dependency to process + + Out-parameters: + deps (dict): will be filled with flattened deps + unpinned_deps (dict): will be filled with unpinned deps + """ + logging.debug('_AddDep(%r)', dep) + + assert dep.name not in deps + deps[dep.name] = dep + + # Detect unpinned deps. + _, revision = gclient_utils.SplitUrlRevision(dep.url) + if not revision or not gclient_utils.IsGitSha(revision): + unpinned_deps[dep.name] = dep + + +def _DepsToLines(deps): + """Converts |deps| dict to list of lines for output.""" + s = ['deps = {'] + for name, dep in sorted(deps.iteritems()): + s.extend([ + ' # %s' % dep.hierarchy(include_url=False), + ' "%s": "%s",' % (name, dep.url), + '', + ]) + s.extend(['}', '']) + return s + + +def _HooksToLines(name, hooks): + """Converts |hooks| list to list of lines for output.""" + s = ['%s = [' % name] + for dep, hook in hooks: + s.extend([ + ' # %s' % dep.hierarchy(include_url=False), + ' {', + ]) + if 'name' in hook: + s.append(' "name": "%s",' % hook['name']) + if 'pattern' in hook: + s.append(' "pattern": "%s",' % hook['pattern']) + # TODO(phajdan.jr): actions may contain paths that need to be adjusted, + # i.e. they may be relative to the dependency path, not solution root. + s.extend( + [' "action": ['] + + [' "%s",' % arg for arg in hook['action']] + + [' ]', ' },', ''] + ) + s.extend([']', '']) + return s + + def CMDgrep(parser, args): """Greps through git repos managed by gclient. diff --git a/testing_support/fake_repos.py b/testing_support/fake_repos.py index 92483fa26e..1c7ae54c5c 100755 --- a/testing_support/fake_repos.py +++ b/testing_support/fake_repos.py @@ -298,7 +298,7 @@ class FakeReposBase(object): class FakeRepos(FakeReposBase): """Implements populateGit().""" - NB_GIT_REPOS = 5 + NB_GIT_REPOS = 6 def populateGit(self): # Testing: @@ -449,6 +449,33 @@ pre_deps_hooks = [ 'origin': 'git/repo_5@3\n', }) + self._commit_git('repo_6', { + 'DEPS': """ +deps = { + 'src/repo2': '%(git_base)srepo_2@%(hash)s', +} +hooks = [ + { + 'pattern': '.', + 'action': ['python', '-c', + 'open(\\'src/git_hooked1\\', \\'w\\').write(\\'git_hooked1\\')'], + }, + { + # Should not be run. + 'pattern': 'nonexistent', + 'action': ['python', '-c', + 'open(\\'src/git_hooked2\\', \\'w\\').write(\\'git_hooked2\\')'], + }, +] +recursedeps = [ + 'src/repo2', +]""" % { + 'git_base': self.git_base, + 'hash': self.git_hashes['repo_2'][1][0][:7] + }, + 'origin': 'git/repo_6@1\n', + }) + class FakeRepoSkiaDEPS(FakeReposBase): """Simulates the Skia DEPS transition in Chrome.""" diff --git a/tests/gclient_smoketest.py b/tests/gclient_smoketest.py index e549786eaf..5deaaebc39 100755 --- a/tests/gclient_smoketest.py +++ b/tests/gclient_smoketest.py @@ -535,6 +535,62 @@ class GClientSmokeGIT(GClientSmokeBase): }) self.check((out, '', 0), results) + def testFlatten(self): + if not self.enabled: + return + + output_deps = os.path.join(self.root_dir, 'DEPS.flattened') + self.assertFalse(os.path.exists(output_deps)) + + self.gclient(['config', self.git_base + 'repo_6', '--name', 'src']) + self.gclient(['sync']) + self.gclient(['flatten', '-v', '-v', '-v', '--output-deps', output_deps]) + + with open(output_deps) as f: + deps_contents = f.read() + + self.assertEqual([ + 'deps = {', + ' # src -> src/repo2 -> foo/bar', + ' "foo/bar": "/repo_3",', + '', + ' # src', + ' "src": "git://127.0.0.1:20000/git/repo_6",', + '', + ' # src -> src/repo2', + ' "src/repo2": "git://127.0.0.1:20000/git/repo_2@%s",' % ( + self.githash('repo_2', 1)[:7]), + '', + '}', + '', + 'hooks = [', + ' # src', + ' {', + ' "pattern": ".",', + ' "action": [', + ' "python",', + ' "-c",', + ' "open(\'src/git_hooked1\', \'w\').write(\'git_hooked1\')",', + ' ]', + ' },', + '', + ' # src', + ' {', + ' "pattern": "nonexistent",', + ' "action": [', + ' "python",', + ' "-c",', + ' "open(\'src/git_hooked2\', \'w\').write(\'git_hooked2\')",', + ' ]', + ' },', + '', + ']', + '', + 'pre_deps_hooks = [', + ']', + '', + ], deps_contents.splitlines()) + class GClientSmokeGITMutates(GClientSmokeBase): """testRevertAndStatus mutates the git repo so move it to its own suite."""