Add support for GCS deps
Also take out GCS calling logic from download_google_storage and
into call_google_storage.
GCS deps look like:
'src/third_party/node/linux': {
'dep_type': 'gcs',
'condition': 'checkout_linux',
'bucket': 'chromium-nodejs/20.11.0',
'object_name': '46795170ff5df9831955f163f6966abde581c8af',
'sha256sum': '887504c37404898ca41b896f448ee6d7fc24179d8fb6a4b79d028ab7e1b7153d',
},
'src/third_party/llvm-build/Release+Asserts': {
'dep_type': 'gcs',
'condition': 'checkout_linux',
'bucket': 'chromium-browser-clang',
'object_name': 'Linux_x64/clang-llvmorg-18-init-17730-gf670112a-2.tar.xz',
'sha256sum': '1e46df9b4e63c074064d75646310cb76be2f19815997a8486987189d80f991e8',
},
Example directory for src/third_party/node/linux after gclient sync:
- tar_file.gz is the downloaded file from GCS.
- node_linux_x64/ is extracted in its path.
- `hash` contains the sha of GCS filename.
```
chromium/src/ ->
third_party/node/linux/ ->
hash, tar_file.gz, node_linux_x64/
```
Bug: b/324418194
Change-Id: Ibcbbff27e211f194ddb8a08494af56570a84a12b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/5299722
Commit-Queue: Stephanie Kim <kimstephanie@google.com>
Reviewed-by: Joanna Wang <jojwang@chromium.org>
changes/22/5299722/31
Stephanie Kim
1 year ago
committed by
LUCI CQ
parent
1ac3eb7b98
commit
3eedee7b55
@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) 2024 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""Download files from Google Storage, given the bucket and file."""
|
||||
|
||||
import optparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import subprocess2
|
||||
|
||||
# Env vars that tempdir can be gotten from; minimally, this
|
||||
# needs to match python's tempfile module and match normal
|
||||
# unix standards.
|
||||
_TEMPDIR_ENV_VARS = ('TMPDIR', 'TEMP', 'TMP')
|
||||
|
||||
GSUTIL_DEFAULT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
'gsutil.py')
|
||||
|
||||
# Maps sys.platform to what we actually want to call them.
|
||||
PLATFORM_MAPPING = {
|
||||
'cygwin': 'win',
|
||||
'darwin': 'mac',
|
||||
'linux': 'linux', # Python 3.3+.
|
||||
'win32': 'win',
|
||||
'aix6': 'aix',
|
||||
'aix7': 'aix',
|
||||
'zos': 'zos',
|
||||
}
|
||||
|
||||
|
||||
def GetNormalizedPlatform():
|
||||
"""Returns the result of sys.platform accounting for cygwin.
|
||||
Under cygwin, this will always return "win32" like the native Python."""
|
||||
if sys.platform == 'cygwin':
|
||||
return 'win32'
|
||||
return sys.platform
|
||||
|
||||
|
||||
# Common utilities
|
||||
class Gsutil(object):
|
||||
"""Call gsutil with some predefined settings. This is a convenience object,
|
||||
and is also immutable.
|
||||
|
||||
HACK: This object is used directly by the external script
|
||||
`<depot_tools>/win_toolchain/get_toolchain_if_necessary.py`
|
||||
"""
|
||||
|
||||
MAX_TRIES = 5
|
||||
RETRY_BASE_DELAY = 5.0
|
||||
RETRY_DELAY_MULTIPLE = 1.3
|
||||
VPYTHON3 = ('vpython3.bat'
|
||||
if GetNormalizedPlatform() == 'win32' else 'vpython3')
|
||||
|
||||
def __init__(self, path, boto_path=None):
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError('GSUtil not found in %s' % path)
|
||||
self.path = path
|
||||
self.boto_path = boto_path
|
||||
|
||||
def get_sub_env(self):
|
||||
env = os.environ.copy()
|
||||
if self.boto_path == os.devnull:
|
||||
env['AWS_CREDENTIAL_FILE'] = ''
|
||||
env['BOTO_CONFIG'] = ''
|
||||
elif self.boto_path:
|
||||
env['AWS_CREDENTIAL_FILE'] = self.boto_path
|
||||
env['BOTO_CONFIG'] = self.boto_path
|
||||
|
||||
if PLATFORM_MAPPING[sys.platform] != 'win':
|
||||
env.update((x, "/tmp") for x in _TEMPDIR_ENV_VARS)
|
||||
|
||||
return env
|
||||
|
||||
def call(self, *args):
|
||||
cmd = [self.VPYTHON3, self.path]
|
||||
cmd.extend(args)
|
||||
return subprocess2.call(cmd, env=self.get_sub_env())
|
||||
|
||||
def check_call(self, *args):
|
||||
cmd = [self.VPYTHON3, self.path]
|
||||
cmd.extend(args)
|
||||
((out, err), code) = subprocess2.communicate(cmd,
|
||||
stdout=subprocess2.PIPE,
|
||||
stderr=subprocess2.PIPE,
|
||||
env=self.get_sub_env())
|
||||
|
||||
out = out.decode('utf-8', 'replace')
|
||||
err = err.decode('utf-8', 'replace')
|
||||
|
||||
# Parse output.
|
||||
status_code_match = re.search('status=([0-9]+)', err)
|
||||
if status_code_match:
|
||||
return (int(status_code_match.group(1)), out, err)
|
||||
if ('ServiceException: 401 Anonymous' in err):
|
||||
return (401, out, err)
|
||||
if ('You are attempting to access protected data with '
|
||||
'no configured credentials.' in err):
|
||||
return (403, out, err)
|
||||
if 'matched no objects' in err or 'No URLs matched' in err:
|
||||
return (404, out, err)
|
||||
return (code, out, err)
|
||||
|
||||
def check_call_with_retries(self, *args):
|
||||
delay = self.RETRY_BASE_DELAY
|
||||
for i in range(self.MAX_TRIES):
|
||||
code, out, err = self.check_call(*args)
|
||||
if not code or i == self.MAX_TRIES - 1:
|
||||
break
|
||||
|
||||
time.sleep(delay)
|
||||
delay *= self.RETRY_DELAY_MULTIPLE
|
||||
|
||||
return code, out, err
|
||||
|
||||
|
||||
def main(args):
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option('-b',
|
||||
'--bucket',
|
||||
help='Google Storage bucket to fetch from.')
|
||||
parser.add_option('-p', '--file', help='Path of file to fetch.')
|
||||
parser.add_option('-o',
|
||||
'--output',
|
||||
help='Path where GCS contents should be downloaded.')
|
||||
parser.add_option('-e', '--boto', help='Specify a custom boto file.')
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
file_url = 'gs://%s/%s' % (options.bucket, options.file)
|
||||
|
||||
# Make sure gsutil exists where we expect it to.
|
||||
if os.path.exists(GSUTIL_DEFAULT_PATH):
|
||||
gsutil = Gsutil(GSUTIL_DEFAULT_PATH, boto_path=options.boto)
|
||||
else:
|
||||
parser.error('gsutil not found in %s, bad depot_tools checkout?' %
|
||||
GSUTIL_DEFAULT_PATH)
|
||||
|
||||
gsutil.check_call('cp', file_url, options.output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(sys.argv))
|
||||
@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env vpython3
|
||||
# Copyright (c) 2024 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""Smoke tests for gclient.py.
|
||||
|
||||
Shell out 'gclient' and run gcs tests.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
from unittest import mock
|
||||
import gclient_smoketest_base
|
||||
import subprocess2
|
||||
|
||||
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
class GClientSmokeGcs(gclient_smoketest_base.GClientSmokeBase):
|
||||
|
||||
def setUp(self):
|
||||
super(GClientSmokeGcs, self).setUp()
|
||||
self.enabled = self.FAKE_REPOS.set_up_git()
|
||||
if not self.enabled:
|
||||
self.skipTest('git fake repos not available')
|
||||
self.env['PATH'] = (os.path.join(ROOT_DIR, 'testing_support') +
|
||||
os.pathsep + self.env['PATH'])
|
||||
|
||||
def testSyncGcs(self):
|
||||
self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
|
||||
self.gclient(['sync'])
|
||||
|
||||
tree = self.mangle_git_tree(('repo_22@1', 'src'))
|
||||
tree.update({
|
||||
'src/another_gcs_dep/hash':
|
||||
'abcd123\n',
|
||||
'src/another_gcs_dep/llvmfile.tar.gz':
|
||||
'tarfile',
|
||||
'src/another_gcs_dep/extracted_dir/extracted_file':
|
||||
'extracted text',
|
||||
'src/gcs_dep/deadbeef':
|
||||
'tarfile',
|
||||
'src/gcs_dep/hash':
|
||||
'abcd123\n',
|
||||
'src/gcs_dep/extracted_dir/extracted_file':
|
||||
'extracted text',
|
||||
})
|
||||
self.assertTree(tree)
|
||||
|
||||
def testConvertGitToGcs(self):
|
||||
self.gclient(['config', self.git_base + 'repo_23', '--name', 'src'])
|
||||
|
||||
# repo_13@1 has src/repo12 as a git dependency.
|
||||
self.gclient([
|
||||
'sync', '-v', '-v', '-v', '--revision',
|
||||
self.githash('repo_23', 1)
|
||||
])
|
||||
|
||||
tree = self.mangle_git_tree(('repo_23@1', 'src'),
|
||||
('repo_12@1', 'src/repo12'))
|
||||
self.assertTree(tree)
|
||||
|
||||
# repo_23@3 has src/repo12 as a gcs dependency.
|
||||
self.gclient([
|
||||
'sync', '-v', '-v', '-v', '--revision',
|
||||
self.githash('repo_23', 3), '--delete_unversioned_trees'
|
||||
])
|
||||
|
||||
tree = self.mangle_git_tree(('repo_23@3', 'src'))
|
||||
tree.update({
|
||||
'src/repo12/extracted_dir/extracted_file': 'extracted text',
|
||||
'src/repo12/hash': 'abcd123\n',
|
||||
'src/repo12/path_to_file.tar.gz': 'tarfile',
|
||||
})
|
||||
self.assertTree(tree)
|
||||
|
||||
def testConvertGcsToGit(self):
|
||||
self.gclient(['config', self.git_base + 'repo_23', '--name', 'src'])
|
||||
|
||||
# repo_13@3 has src/repo12 as a cipd dependency.
|
||||
self.gclient([
|
||||
'sync', '-v', '-v', '-v', '--revision',
|
||||
self.githash('repo_23', 3), '--delete_unversioned_trees'
|
||||
])
|
||||
|
||||
tree = self.mangle_git_tree(('repo_23@3', 'src'))
|
||||
tree.update({
|
||||
'src/repo12/extracted_dir/extracted_file': 'extracted text',
|
||||
'src/repo12/hash': 'abcd123\n',
|
||||
'src/repo12/path_to_file.tar.gz': 'tarfile',
|
||||
})
|
||||
self.assertTree(tree)
|
||||
|
||||
# repo_23@1 has src/repo12 as a git dependency.
|
||||
self.gclient([
|
||||
'sync', '-v', '-v', '-v', '--revision',
|
||||
self.githash('repo_23', 1)
|
||||
])
|
||||
|
||||
tree = self.mangle_git_tree(('repo_23@1', 'src'),
|
||||
('repo_12@1', 'src/repo12'))
|
||||
tree.update({
|
||||
'src/repo12/extracted_dir/extracted_file': 'extracted text',
|
||||
'src/repo12/hash': 'abcd123\n',
|
||||
'src/repo12/path_to_file.tar.gz': 'tarfile',
|
||||
})
|
||||
self.assertTree(tree)
|
||||
|
||||
def testRevInfo(self):
|
||||
self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
|
||||
self.gclient(['sync'])
|
||||
results = self.gclient(['revinfo'])
|
||||
out = ('src: %(base)srepo_22\n'
|
||||
'src/another_gcs_dep: gs://456bucket/Linux/llvmfile.tar.gz\n'
|
||||
'src/gcs_dep: gs://123bucket/deadbeef\n' % {
|
||||
'base': self.git_base,
|
||||
})
|
||||
self.check((out, '', 0), results)
|
||||
|
||||
def testRevInfoActual(self):
|
||||
self.gclient(['config', self.git_base + 'repo_22', '--name', 'src'])
|
||||
self.gclient(['sync'])
|
||||
results = self.gclient(['revinfo', '--actual'])
|
||||
out = (
|
||||
'src: %(base)srepo_22@%(hash1)s\n'
|
||||
'src/another_gcs_dep: gs://456bucket/Linux/llvmfile.tar.gz@None\n'
|
||||
'src/gcs_dep: gs://123bucket/deadbeef@None\n' % {
|
||||
'base': self.git_base,
|
||||
'hash1': self.githash('repo_22', 1),
|
||||
})
|
||||
self.check((out, '', 0), results)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if '-v' in sys.argv:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
unittest.main()
|
||||
Loading…
Reference in New Issue