Add a script to copy GCS packages from one DEPS file to another
This script will be used to roll downstream GCS packages. Bug: 358435510 Change-Id: I7ea1229eb4e8c4c590cad336573f9d24662f5730 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5765858 Reviewed-by: Joanna Wang <jojwang@chromium.org> Commit-Queue: Josip Sokcevic <sokcevic@chromium.org>changes/58/5765858/9
Josip Sokcevic
9 months ago
committed by
LUCI CQ
parent
ec800aa077
commit
bed17582bf
@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""This scripts copies DEPS package information from one source onto
|
||||
destination.
|
||||
|
||||
If the destination doesn't have packages, the script errors out.
|
||||
|
||||
Example usage:
|
||||
roll_downstream_gcs_deps.py \
|
||||
--source some/repo/DEPS \
|
||||
--destination some/downstream/repo/DEPS \
|
||||
--package src/build/linux/debian_bullseye_amd64-sysroot \
|
||||
--package src/build/linux/debian_bullseye_arm64-sysroot
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import sys
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def _get_deps(deps_ast: ast.Module) -> Dict[str, ast.Dict]:
|
||||
"""Searches for the deps dict in a DEPS file AST.
|
||||
|
||||
Args:
|
||||
deps_ast: AST of the DEPS file.
|
||||
|
||||
Raises:
|
||||
Exception: If the deps dict is not found.
|
||||
|
||||
Returns:
|
||||
The deps dict.
|
||||
"""
|
||||
for statement in deps_ast.body:
|
||||
if not isinstance(statement, ast.Assign):
|
||||
continue
|
||||
if len(statement.targets) != 1:
|
||||
continue
|
||||
target = statement.targets[0]
|
||||
if not isinstance(target, ast.Name):
|
||||
continue
|
||||
if target.id != 'deps':
|
||||
continue
|
||||
if not isinstance(statement.value, ast.Dict):
|
||||
continue
|
||||
deps = {}
|
||||
for key, value in zip(statement.value.keys, statement.value.values):
|
||||
if not isinstance(key, ast.Constant):
|
||||
continue
|
||||
deps[key.value] = value
|
||||
return deps
|
||||
raise Exception('no deps found')
|
||||
|
||||
|
||||
def _get_gcs_object_list_ast(package_ast: ast.Dict) -> ast.List:
|
||||
"""Searches for the objects list in a GCS package AST.
|
||||
|
||||
Args:
|
||||
package_ast: AST of the GCS package.
|
||||
|
||||
Raises:
|
||||
Exception: If the package is not a GCS package.
|
||||
|
||||
Returns:
|
||||
AST of the objects list.
|
||||
"""
|
||||
is_gcs = False
|
||||
result = None
|
||||
for key, value in zip(package_ast.keys, package_ast.values):
|
||||
if not isinstance(key, ast.Constant):
|
||||
continue
|
||||
if key.value == 'dep_type' and isinstance(
|
||||
value, ast.Constant) and value.value == 'gcs':
|
||||
is_gcs = True
|
||||
if key.value == 'objects' and isinstance(value, ast.List):
|
||||
result = value
|
||||
|
||||
assert is_gcs, 'Not a GCS dependency!'
|
||||
assert result, 'No objects found!'
|
||||
return result
|
||||
|
||||
|
||||
def _replace_ast(destination: str, dest_ast: ast.Module, source: str,
|
||||
source_ast: ast.Module) -> str:
|
||||
"""Replaces the content of dest_ast with the content of the
|
||||
same package in source_ast.
|
||||
|
||||
Args:
|
||||
destination: Destination DEPS file content.
|
||||
dest_ast: AST in the destination DEPS file that will be replaced.
|
||||
source: Source DEPS file content.
|
||||
source_ast: AST in the source DEPS file that will replace content of
|
||||
destination.
|
||||
|
||||
Returns:
|
||||
Content of destination DEPS file with replaced content.
|
||||
"""
|
||||
source_lines = source.splitlines()
|
||||
lines = destination.splitlines()
|
||||
# Copy all lines before the replaced AST.
|
||||
result = '\n'.join(lines[:dest_ast.lineno - 1]) + '\n'
|
||||
|
||||
# Partially copy the line content before AST's value.
|
||||
result += lines[dest_ast.lineno - 1][:dest_ast.col_offset]
|
||||
|
||||
# Copy data from source AST.
|
||||
if source_ast.lineno == source_ast.end_lineno:
|
||||
# Starts and ends on the same line.
|
||||
result += source_lines[
|
||||
source_ast.lineno -
|
||||
1][source_ast.col_offset:source_ast.end_col_offset]
|
||||
else:
|
||||
# Copy multiline content from source. The first line and the last line
|
||||
# of source AST should be partially copied as `result` has a partial
|
||||
# line from `destination`.
|
||||
|
||||
# Partially copy the first line of source AST.
|
||||
result += source_lines[source_ast.lineno -
|
||||
1][source_ast.col_offset:] + '\n'
|
||||
# Copy content in the middle.
|
||||
result += '\n'.join(
|
||||
source_lines[source_ast.lineno:source_ast.end_lineno - 1]) + '\n'
|
||||
# Partially copy the last line of source AST.
|
||||
result += source_lines[source_ast.end_lineno -
|
||||
1][:source_ast.end_col_offset]
|
||||
|
||||
# Copy the rest of the line after the package value.
|
||||
result += lines[dest_ast.end_lineno - 1][dest_ast.end_col_offset:] + '\n'
|
||||
|
||||
# Copy the rest of the lines after the package value.
|
||||
result += '\n'.join(lines[dest_ast.end_lineno:])
|
||||
# Add trailing newline
|
||||
if destination.endswith('\n'):
|
||||
result += '\n'
|
||||
return result
|
||||
|
||||
|
||||
def copy_packages(source: str, destination: str, packages: List[str]) -> str:
|
||||
"""Copies GCS packages from source to destination.
|
||||
|
||||
Args:
|
||||
source: Source DEPS file content.
|
||||
destination: Destination DEPS file content.
|
||||
packages: List of GCS packages to copy. Only objects are copied.
|
||||
|
||||
Returns:
|
||||
Destination DEPS file content with packages copied.
|
||||
"""
|
||||
source_deps = _get_deps(ast.parse(source, mode='exec'))
|
||||
for package in packages:
|
||||
if package not in source_deps:
|
||||
raise Exception('Package %s not found in source' % package)
|
||||
dest_deps = _get_deps(ast.parse(destination, mode='exec'))
|
||||
if package not in dest_deps:
|
||||
raise Exception('Package %s not found in destination' % package)
|
||||
destination = _replace_ast(
|
||||
destination, _get_gcs_object_list_ast(dest_deps[package]), source,
|
||||
_get_gcs_object_list_ast(source_deps[package]))
|
||||
|
||||
return destination
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--source',
|
||||
required=True,
|
||||
help='Source DEPS file where content will be copied '
|
||||
'from')
|
||||
parser.add_argument('--package',
|
||||
action='append',
|
||||
required=True,
|
||||
help='List of DEPS packages to update')
|
||||
parser.add_argument('--destination',
|
||||
required=True,
|
||||
help='Destination DEPS file, where content will be '
|
||||
'saved')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.package:
|
||||
parser.error('No packages specified to roll, aborting...')
|
||||
|
||||
with open(args.source) as f:
|
||||
source_content = f.read()
|
||||
|
||||
with open(args.destination) as f:
|
||||
destination_content = f.read()
|
||||
|
||||
new_content = copy_packages(source_content, destination_content,
|
||||
args.package)
|
||||
|
||||
with open(args.destination, 'w') as f:
|
||||
f.write(new_content)
|
||||
|
||||
print('Run:')
|
||||
print(' Destination DEPS file updated. You still need to create and '
|
||||
'upload a change.')
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.insert(0, ROOT_DIR)
|
||||
|
||||
import roll_downstream_gcs_deps
|
||||
|
||||
|
||||
class CopyPackageTest(unittest.TestCase):
|
||||
|
||||
def testNoDepsToRoll(self):
|
||||
with self.assertRaises(Exception):
|
||||
roll_downstream_gcs_deps.copy_packages('', '', ['foo'])
|
||||
|
||||
with self.assertRaises(Exception):
|
||||
roll_downstream_gcs_deps.copy_packages('deps = {"foo": ""}',
|
||||
'deps = {}', ['foo'])
|
||||
|
||||
with self.assertRaises(Exception):
|
||||
roll_downstream_gcs_deps.copy_packages('deps = {"foo": ""}',
|
||||
'deps = {"bar": ""}',
|
||||
['foo'])
|
||||
|
||||
def testNoGCSDeps(self):
|
||||
source_deps = '''
|
||||
deps = {
|
||||
"foo": {
|
||||
'dep_type': 'unknown',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'foo-v2.tar.xz',
|
||||
'sha256sum': '1111111111111111111111111111111111111111111111111111111111111111',
|
||||
'size_bytes': 101,
|
||||
'generation': 901,
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
'''
|
||||
destination_deps = '''
|
||||
deps = {
|
||||
"foo": {
|
||||
'dep_type': 'unknown',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'foo.tar.xz',
|
||||
'sha256sum': '0000000000000000000000000000000000000000000000000000000000000000',
|
||||
'size_bytes': 100,
|
||||
'generation': 900,
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
'''
|
||||
with self.assertRaises(Exception):
|
||||
roll_downstream_gcs_deps.copy_packages(source_deps,
|
||||
destination_deps, ['foo'])
|
||||
|
||||
def testObjectInLineUpdate(self):
|
||||
source_deps = '''
|
||||
deps = {
|
||||
"foo": {
|
||||
'dep_type': 'gcs',
|
||||
'bucket': 'foo',
|
||||
'condition': 'deps source condition',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'foo-v2.tar.xz',
|
||||
'sha256sum': '1111111111111111111111111111111111111111111111111111111111111111',
|
||||
'size_bytes': 101,
|
||||
'generation': 901,
|
||||
'condition': 'host_os == "linux" and non_git_source and new',
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
'''
|
||||
destination_deps = '''
|
||||
deps = {
|
||||
"other": "preserved",
|
||||
"foo": {
|
||||
'dep_type': 'gcs',
|
||||
'bucket': 'foo',
|
||||
'condition': 'deps dest condition',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'foo.tar.xz',
|
||||
'sha256sum': '0000000000000000000000000000000000000000000000000000000000000000',
|
||||
'size_bytes': 100,
|
||||
'generation': 900,
|
||||
'condition': 'host_os == "linux" and non_git_source',
|
||||
},
|
||||
]
|
||||
},
|
||||
"another": "preserved",
|
||||
}
|
||||
'''
|
||||
expected_deps = '''
|
||||
deps = {
|
||||
"other": "preserved",
|
||||
"foo": {
|
||||
'dep_type': 'gcs',
|
||||
'bucket': 'foo',
|
||||
'condition': 'deps dest condition',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'foo-v2.tar.xz',
|
||||
'sha256sum': '1111111111111111111111111111111111111111111111111111111111111111',
|
||||
'size_bytes': 101,
|
||||
'generation': 901,
|
||||
'condition': 'host_os == "linux" and non_git_source and new',
|
||||
},
|
||||
]
|
||||
},
|
||||
"another": "preserved",
|
||||
}
|
||||
'''
|
||||
result = roll_downstream_gcs_deps.copy_packages(source_deps,
|
||||
destination_deps,
|
||||
['foo'])
|
||||
self.assertEqual(result, expected_deps)
|
||||
|
||||
def testGCSRustPackageNewPlatform(self):
|
||||
source_deps = '''
|
||||
deps = {
|
||||
'src/third_party/rust-toolchain': {
|
||||
'dep_type': 'gcs',
|
||||
'bucket': 'chromium-browser-clang',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'Linux_x64/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
|
||||
'sha256sum': '560c02da5300f40441992ef639d83cee96cae3584c3d398704fdb2f02e475bbf',
|
||||
'size_bytes': 152024840,
|
||||
'generation': 1722663990116408,
|
||||
'condition': 'host_os == "linux" and non_git_source',
|
||||
},
|
||||
{
|
||||
'object_name': 'Mac/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
|
||||
'sha256sum': '9f39154b4337438fd170e729ed2ae4c978b22f11708d683c28265bd096df17a5',
|
||||
'size_bytes': 144459260,
|
||||
'generation': 1722663991651609,
|
||||
'condition': 'host_os == "mac" and host_cpu == "x64"',
|
||||
},
|
||||
{
|
||||
'object_name': 'Mac_arm64/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
|
||||
'sha256sum': '4b89cf125ffa39e8fc74f01ec3beeb632fd3069478d8c6cc4fcae506b4917151',
|
||||
'size_bytes': 135571272,
|
||||
'generation': 1722663993205996,
|
||||
'condition': 'host_os == "mac" and host_cpu == "arm64"',
|
||||
},
|
||||
{
|
||||
'object_name': 'Win/rust-toolchain-595316b4006932405a63862d8fe65f71a6356293-3-llvmorg-20-init-1009-g7088a5ed.tar.xz',
|
||||
'sha256sum': '3f6a1a87695902062a6575632552b9f2cbbbcda1907fe3232f49b8ea29baecf5',
|
||||
'size_bytes': 208844028,
|
||||
'generation': 1722663994756449,
|
||||
'condition': 'host_os == "win"',
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
'''
|
||||
destination_deps = '''
|
||||
deps = {
|
||||
'src/third_party/rust-toolchain': {
|
||||
'dep_type': 'gcs',
|
||||
'bucket': 'chromium-browser-clang',
|
||||
'objects': [
|
||||
{
|
||||
'object_name': 'Linux_x64/rust-toolchain-0000000000000000000000000000000000000000-0.tar.xz',
|
||||
'sha256sum': '0000000000000000000000000000000000000000000000000000000000000000',
|
||||
'size_bytes': 123,
|
||||
'generation': 987,
|
||||
'condition': 'other condition',
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
'''
|
||||
result = roll_downstream_gcs_deps.copy_packages(
|
||||
source_deps, destination_deps, ['src/third_party/rust-toolchain'])
|
||||
self.assertEqual(result, source_deps)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
level = logging.DEBUG if '-v' in sys.argv else logging.FATAL
|
||||
logging.basicConfig(level=level,
|
||||
format='%(asctime).19s %(levelname)s %(filename)s:'
|
||||
'%(lineno)s %(message)s')
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue