From bed17582bfa1bbf05f445adead5ddf3b383394a9 Mon Sep 17 00:00:00 2001 From: Josip Sokcevic Date: Thu, 8 Aug 2024 22:52:22 +0000 Subject: [PATCH] Add a script to copy GCS packages from one DEPS file to another This script will be used to roll downstream GCS packages. Bug: 358435510 Change-Id: I7ea1229eb4e8c4c590cad336573f9d24662f5730 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5765858 Reviewed-by: Joanna Wang Commit-Queue: Josip Sokcevic --- roll_downstream_gcs_deps.py | 204 +++++++++++++++++++++++++ tests/roll_downstream_gcs_deps_test.py | 197 ++++++++++++++++++++++++ 2 files changed, 401 insertions(+) create mode 100755 roll_downstream_gcs_deps.py create mode 100755 tests/roll_downstream_gcs_deps_test.py diff --git a/roll_downstream_gcs_deps.py b/roll_downstream_gcs_deps.py new file mode 100755 index 000000000..7d5e6a1d5 --- /dev/null +++ b/roll_downstream_gcs_deps.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""This scripts copies DEPS package information from one source onto +destination. + +If the destination doesn't have packages, the script errors out. + +Example usage: + roll_downstream_gcs_deps.py \ + --source some/repo/DEPS \ + --destination some/downstream/repo/DEPS \ + --package src/build/linux/debian_bullseye_amd64-sysroot \ + --package src/build/linux/debian_bullseye_arm64-sysroot + +""" + +import argparse +import ast +import sys +from typing import Dict, List + + +def _get_deps(deps_ast: ast.Module) -> Dict[str, ast.Dict]: + """Searches for the deps dict in a DEPS file AST. + + Args: + deps_ast: AST of the DEPS file. + + Raises: + Exception: If the deps dict is not found. + + Returns: + The deps dict. + """ + for statement in deps_ast.body: + if not isinstance(statement, ast.Assign): + continue + if len(statement.targets) != 1: + continue + target = statement.targets[0] + if not isinstance(target, ast.Name): + continue + if target.id != 'deps': + continue + if not isinstance(statement.value, ast.Dict): + continue + deps = {} + for key, value in zip(statement.value.keys, statement.value.values): + if not isinstance(key, ast.Constant): + continue + deps[key.value] = value + return deps + raise Exception('no deps found') + + +def _get_gcs_object_list_ast(package_ast: ast.Dict) -> ast.List: + """Searches for the objects list in a GCS package AST. + + Args: + package_ast: AST of the GCS package. + + Raises: + Exception: If the package is not a GCS package. + + Returns: + AST of the objects list. + """ + is_gcs = False + result = None + for key, value in zip(package_ast.keys, package_ast.values): + if not isinstance(key, ast.Constant): + continue + if key.value == 'dep_type' and isinstance( + value, ast.Constant) and value.value == 'gcs': + is_gcs = True + if key.value == 'objects' and isinstance(value, ast.List): + result = value + + assert is_gcs, 'Not a GCS dependency!' + assert result, 'No objects found!' + return result + + +def _replace_ast(destination: str, dest_ast: ast.Module, source: str, + source_ast: ast.Module) -> str: + """Replaces the content of dest_ast with the content of the + same package in source_ast. + + Args: + destination: Destination DEPS file content. + dest_ast: AST in the destination DEPS file that will be replaced. + source: Source DEPS file content. + source_ast: AST in the source DEPS file that will replace content of + destination. + + Returns: + Content of destination DEPS file with replaced content. + """ + source_lines = source.splitlines() + lines = destination.splitlines() + # Copy all lines before the replaced AST. + result = '\n'.join(lines[:dest_ast.lineno - 1]) + '\n' + + # Partially copy the line content before AST's value. + result += lines[dest_ast.lineno - 1][:dest_ast.col_offset] + + # Copy data from source AST. + if source_ast.lineno == source_ast.end_lineno: + # Starts and ends on the same line. + result += source_lines[ + source_ast.lineno - + 1][source_ast.col_offset:source_ast.end_col_offset] + else: + # Copy multiline content from source. The first line and the last line + # of source AST should be partially copied as `result` has a partial + # line from `destination`. + + # Partially copy the first line of source AST. + result += source_lines[source_ast.lineno - + 1][source_ast.col_offset:] + '\n' + # Copy content in the middle. + result += '\n'.join( + source_lines[source_ast.lineno:source_ast.end_lineno - 1]) + '\n' + # Partially copy the last line of source AST. + result += source_lines[source_ast.end_lineno - + 1][:source_ast.end_col_offset] + + # Copy the rest of the line after the package value. + result += lines[dest_ast.end_lineno - 1][dest_ast.end_col_offset:] + '\n' + + # Copy the rest of the lines after the package value. + result += '\n'.join(lines[dest_ast.end_lineno:]) + # Add trailing newline + if destination.endswith('\n'): + result += '\n' + return result + + +def copy_packages(source: str, destination: str, packages: List[str]) -> str: + """Copies GCS packages from source to destination. + + Args: + source: Source DEPS file content. + destination: Destination DEPS file content. + packages: List of GCS packages to copy. Only objects are copied. + + Returns: + Destination DEPS file content with packages copied. + """ + source_deps = _get_deps(ast.parse(source, mode='exec')) + for package in packages: + if package not in source_deps: + raise Exception('Package %s not found in source' % package) + dest_deps = _get_deps(ast.parse(destination, mode='exec')) + if package not in dest_deps: + raise Exception('Package %s not found in destination' % package) + destination = _replace_ast( + destination, _get_gcs_object_list_ast(dest_deps[package]), source, + _get_gcs_object_list_ast(source_deps[package])) + + return destination + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', + required=True, + help='Source DEPS file where content will be copied ' + 'from') + parser.add_argument('--package', + action='append', + required=True, + help='List of DEPS packages to update') + parser.add_argument('--destination', + required=True, + help='Destination DEPS file, where content will be ' + 'saved') + args = parser.parse_args() + + if not args.package: + parser.error('No packages specified to roll, aborting...') + + with open(args.source) as f: + source_content = f.read() + + with open(args.destination) as f: + destination_content = f.read() + + new_content = copy_packages(source_content, destination_content, + args.package) + + with open(args.destination, 'w') as f: + f.write(new_content) + + print('Run:') + print(' Destination DEPS file updated. You still need to create and ' + 'upload a change.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/roll_downstream_gcs_deps_test.py b/tests/roll_downstream_gcs_deps_test.py new file mode 100755 index 000000000..e161bda91 --- /dev/null +++ b/tests/roll_downstream_gcs_deps_test.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import logging +import os +import sys +import unittest + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT_DIR) + +import roll_downstream_gcs_deps + + +class CopyPackageTest(unittest.TestCase): + + def testNoDepsToRoll(self): + with self.assertRaises(Exception): + roll_downstream_gcs_deps.copy_packages('', '', ['foo']) + + with self.assertRaises(Exception): + roll_downstream_gcs_deps.copy_packages('deps = {"foo": ""}', + 'deps = {}', ['foo']) + + with self.assertRaises(Exception): + roll_downstream_gcs_deps.copy_packages('deps = {"foo": ""}', + 'deps = {"bar": ""}', + ['foo']) + + def testNoGCSDeps(self): + source_deps = ''' +deps = { + "foo": { + 'dep_type': 'unknown', + 'objects': [ + { + 'object_name': 'foo-v2.tar.xz', + 'sha256sum': '1111111111111111111111111111111111111111111111111111111111111111', + 'size_bytes': 101, + 'generation': 901, + }, + ] + }, +} +''' + destination_deps = ''' +deps = { + "foo": { + 'dep_type': 'unknown', + 'objects': [ + { + 'object_name': 'foo.tar.xz', + 'sha256sum': '0000000000000000000000000000000000000000000000000000000000000000', + 'size_bytes': 100, + 'generation': 900, + }, + ] + }, +} +''' + with self.assertRaises(Exception): + roll_downstream_gcs_deps.copy_packages(source_deps, + destination_deps, ['foo']) + + def testObjectInLineUpdate(self): + source_deps = ''' +deps = { + "foo": { + 'dep_type': 'gcs', + 'bucket': 'foo', + 'condition': 'deps source condition', + 'objects': [ + { + 'object_name': 'foo-v2.tar.xz', + 'sha256sum': '1111111111111111111111111111111111111111111111111111111111111111', + 'size_bytes': 101, + 'generation': 901, + 'condition': 'host_os == "linux" and non_git_source and new', + }, + ] + }, +} +''' + destination_deps = ''' +deps = { + "other": "preserved", + "foo": { + 'dep_type': 'gcs', + 'bucket': 'foo', + 'condition': 'deps dest condition', + 'objects': [ + { + 'object_name': 'foo.tar.xz', + 'sha256sum': '0000000000000000000000000000000000000000000000000000000000000000', + 'size_bytes': 100, + 'generation': 900, + 'condition': 'host_os == "linux" and non_git_source', + }, + ] + }, + "another": "preserved", +} +''' + expected_deps = ''' +deps = { + "other": "preserved", + "foo": { + 'dep_type': 'gcs', + 'bucket': 'foo', + 'condition': 'deps dest condition', + 'objects': [ + { + 'object_name': 'foo-v2.tar.xz', + 'sha256sum': '1111111111111111111111111111111111111111111111111111111111111111', + 'size_bytes': 101, + 'generation': 901, + 'condition': 'host_os == "linux" and non_git_source and new', + }, + ] + }, + "another": "preserved", +} +''' + result = roll_downstream_gcs_deps.copy_packages(source_deps, + destination_deps, + ['foo']) + self.assertEqual(result, expected_deps) + + def testGCSRustPackageNewPlatform(self): + source_deps = ''' +deps = { + 'src/third_party/rust-toolchain': { + 'dep_type': 'gcs', + 'bucket': 'chromium-browser-clang', + 'objects': [ + { + 'object_name': 'Linux_x64/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz', + 'sha256sum': '560c02da5300f40441992ef639d83cee96cae3584c3d398704fdb2f02e475bbf', + 'size_bytes': 152024840, + 'generation': 1722663990116408, + 'condition': 'host_os == "linux" and non_git_source', + }, + { + 'object_name': 'Mac/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz', + 'sha256sum': '9f39154b4337438fd170e729ed2ae4c978b22f11708d683c28265bd096df17a5', + 'size_bytes': 144459260, + 'generation': 1722663991651609, + 'condition': 'host_os == "mac" and host_cpu == "x64"', + }, + { + 'object_name': 'Mac_arm64/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz', + 'sha256sum': '4b89cf125ffa39e8fc74f01ec3beeb632fd3069478d8c6cc4fcae506b4917151', + 'size_bytes': 135571272, + 'generation': 1722663993205996, + 'condition': 'host_os == "mac" and host_cpu == "arm64"', + }, + { + 'object_name': 'Win/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz', + 'sha256sum': '3f6a1a87695902062a6575632552b9f2cbbbcda1907fe3232f49b8ea29baecf5', + 'size_bytes': 208844028, + 'generation': 1722663994756449, + 'condition': 'host_os == "win"', + }, + ], + }, +} +''' + destination_deps = ''' +deps = { + 'src/third_party/rust-toolchain': { + 'dep_type': 'gcs', + 'bucket': 'chromium-browser-clang', + 'objects': [ + { + 'object_name': 'Linux_x64/rust-toolchain-0000000000000000000000000000000000000000-0.tar.xz', + 'sha256sum': '0000000000000000000000000000000000000000000000000000000000000000', + 'size_bytes': 123, + 'generation': 987, + 'condition': 'other condition', + }, + ], + }, +} +''' + result = roll_downstream_gcs_deps.copy_packages( + source_deps, destination_deps, ['src/third_party/rust-toolchain']) + self.assertEqual(result, source_deps) + + +if __name__ == '__main__': + level = logging.DEBUG if '-v' in sys.argv else logging.FATAL + logging.basicConfig(level=level, + format='%(asctime).19s %(levelname)s %(filename)s:' + '%(lineno)s %(message)s') + unittest.main()