Roll cq_client.
From infra_internal, revisions 6f979a..cd0150f. TBR=sergiyb@chromium.org,machenbach@chromium.org BUG= Review URL: https://codereview.chromium.org/1697053003 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@298789 0039d316-1c4b-4281-b951-d872f2087c98changes/01/332501/1
tandrii@chromium.org
9 years ago
parent
a18cfe6573
commit
a1afbb986d
@ -1,29 +1,55 @@
|
||||
# CQ Client Library.
|
||||
|
||||
This directory contains CQ client library to be distributed to other repos. If
|
||||
you need to modify some files in this directory, please make sure that you are
|
||||
changing the canonical version of the source code and not one of the copies,
|
||||
which should only be updated as a whole using Glyco (when available, see
|
||||
[chromium issue 489420](http://crbug.com/489420)).
|
||||
|
||||
The canonical version is located at `https://chrome-internal.googlesource.com/
|
||||
infra/infra_internal/+/master/commit_queue/cq_client`.
|
||||
The canonical version is located at
|
||||
[https://chrome-internal.googlesource.com/infra/infra_internal/+/master/commit_queue/cq_client]().
|
||||
|
||||
When modifying cq.proto, consider adding checks to validator in
|
||||
[https://chrome-internal.googlesource.com/infra/infra_internal/+/master/appengine/commit_queue/src/commitqueue/validate.go]().
|
||||
|
||||
|
||||
## Generation of Python and Go bindings
|
||||
|
||||
### tl;dr
|
||||
|
||||
make
|
||||
|
||||
|
||||
### Details
|
||||
|
||||
All commands below assume you are working in a standard infra_internal gclient
|
||||
checkout (e.g., after you ran `mkdir src && cd src && fetch infra_internal`) and
|
||||
are in current directory of this README.md (that is, in
|
||||
`cd infra_internal/commit_queue/cq_client`).
|
||||
|
||||
You'll need to use protoc version 2.6.1 and
|
||||
recent golang/protobuf package. Sadly, the latter has neither tags nor versions.
|
||||
To generate Python's `cq_pb2.py` you'll need to get and `protoc` of version
|
||||
**2.6.1**. You can get it by `make py-prepare`.
|
||||
|
||||
make py
|
||||
|
||||
You can get protobuf by downloading archive from
|
||||
https://github.com/google/protobuf/tree/v2.6.1 and manually building it. As for
|
||||
golang compiler, if you have go configured, just
|
||||
To generate Golang's protobuf file `cq.pb.go`, you'll need to bootstrap
|
||||
infra/infra repository and go utilities `make go-prepare`.
|
||||
|
||||
go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
|
||||
make go
|
||||
|
||||
TODO(tandrii,sergiyb): decide how to pin the go protobuf generator.
|
||||
## Notes
|
||||
|
||||
To generate `cq_pb2.py` and `cq.pb.go`:
|
||||
1. Please make sure to use proto3-compatible yntax, e.g. no default
|
||||
values, no required fields. As of this writing (Jan 2016),
|
||||
the Go protobuf compiler has been upgraded to 3.0.0. So, if you can generate go
|
||||
bindings, all is good.
|
||||
|
||||
cd commit_queue/cq_client
|
||||
protoc cq.proto --python_out $(pwd) --go_out $(pwd)
|
||||
2. If after generating Python binding, CQ tests fail with:
|
||||
|
||||
Additionally, please make sure to use proto3-compatible syntax, e.g. no default
|
||||
values, no required fields. Ideally, we should use proto3 generator already,
|
||||
however alpha version thereof is still unstable.
|
||||
TypeError: __init__() got an unexpected keyword argument 'syntax'
|
||||
|
||||
You've probably used 3.0.0 protoc generator. We should eventually switch to 3x
|
||||
Python version as well, but it requires upgrading infra's Python ENV to newer
|
||||
package. See [bootstrap/README.md](../../bootstrap/README.md) for more
|
||||
information. We may end up deprecating Python before all infra's Python code
|
||||
can be moved to protobuf v3.
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
|
||||
|
||||
@ -1,66 +0,0 @@
|
||||
# Copyright 2015 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""Unit tests for tools/validate_config.py."""
|
||||
|
||||
import mock
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from cq_client import cq_pb2
|
||||
from cq_client import validate_config
|
||||
|
||||
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
class TestValidateConfig(unittest.TestCase):
|
||||
def test_is_valid_rietveld(self):
|
||||
with open(os.path.join(TEST_DIR, 'cq_rietveld.cfg'), 'r') as test_config:
|
||||
self.assertTrue(validate_config.IsValid(test_config.read()))
|
||||
|
||||
def test_is_valid_gerrit(self):
|
||||
with open(os.path.join(TEST_DIR, 'cq_gerrit.cfg'), 'r') as test_config:
|
||||
self.assertTrue(validate_config.IsValid(test_config.read()))
|
||||
|
||||
def test_one_codereview(self):
|
||||
with open(os.path.join(TEST_DIR, 'cq_gerrit.cfg'), 'r') as gerrit_config:
|
||||
data = gerrit_config.read()
|
||||
data += '\n'.join([
|
||||
'rietveld{',
|
||||
'url: "https://blabla.com"',
|
||||
'}'
|
||||
])
|
||||
self.assertFalse(validate_config.IsValid(data))
|
||||
|
||||
def test_has_field(self):
|
||||
config = cq_pb2.Config()
|
||||
|
||||
self.assertFalse(validate_config._HasField(config, 'version'))
|
||||
config.version = 1
|
||||
self.assertTrue(validate_config._HasField(config, 'version'))
|
||||
|
||||
self.assertFalse(validate_config._HasField(
|
||||
config, 'rietveld.project_bases'))
|
||||
config.rietveld.project_bases.append('foo://bar')
|
||||
self.assertTrue(validate_config._HasField(
|
||||
config, 'rietveld.project_bases'))
|
||||
|
||||
self.assertFalse(validate_config._HasField(
|
||||
config, 'verifiers.try_job.buckets'))
|
||||
self.assertFalse(validate_config._HasField(
|
||||
config, 'verifiers.try_job.buckets.name'))
|
||||
|
||||
bucket = config.verifiers.try_job.buckets.add()
|
||||
bucket.name = 'tryserver.chromium.linux'
|
||||
|
||||
|
||||
self.assertTrue(validate_config._HasField(
|
||||
config, 'verifiers.try_job.buckets'))
|
||||
self.assertTrue(validate_config._HasField(
|
||||
config, 'verifiers.try_job.buckets.name'))
|
||||
|
||||
config.verifiers.try_job.buckets.add()
|
||||
self.assertFalse(validate_config._HasField(
|
||||
config, 'verifiers.try_job.buckets.name'))
|
||||
@ -1,125 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2015 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""CQ config validation library."""
|
||||
|
||||
import argparse
|
||||
# This file was originally copied together with the cq_client library from the
|
||||
# internal commit_queue repository and then modified to import protobuf26
|
||||
# instead of google.protobuf to prevent conflicts with a different version of
|
||||
# google.protobuf that some users of depot_tools have installed. If you need to
|
||||
# update this file, please make similar changes again and add this comment back.
|
||||
# More details on why we chose to rename the package can be found in the file
|
||||
# depot_tools/third_party/protobuf26/README.chromium.
|
||||
import protobuf26 as protobuf
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
|
||||
from cq_client import cq_pb2
|
||||
|
||||
|
||||
REQUIRED_FIELDS = [
|
||||
'version',
|
||||
'verifiers',
|
||||
'cq_name',
|
||||
]
|
||||
|
||||
LEGACY_FIELDS = [
|
||||
'svn_repo_url',
|
||||
]
|
||||
|
||||
EMAIL_REGEXP = '^[^@]+@[^@]+\.[^@]+$'
|
||||
|
||||
|
||||
def _HasField(message, field_path):
|
||||
"""Checks that at least one field with given path exist in the proto message.
|
||||
|
||||
This function correctly handles repeated fields and will make sure that each
|
||||
repeated field will have required sub-path, e.g. if 'abc' is a repeated field
|
||||
and field_path is 'abc.def', then the function will only return True when each
|
||||
entry for 'abc' will contain at least one value for 'def'.
|
||||
|
||||
Args:
|
||||
message (google.protobuf.message.Message): Protocol Buffer message to check.
|
||||
field_path (string): Path to the target field separated with ".".
|
||||
|
||||
Return:
|
||||
True if at least one such field is explicitly set in the message.
|
||||
"""
|
||||
path_parts = field_path.split('.', 1)
|
||||
field_name = path_parts[0]
|
||||
sub_path = path_parts[1] if len(path_parts) == 2 else None
|
||||
|
||||
field_labels = {fd.name: fd.label for fd in message.DESCRIPTOR.fields}
|
||||
repeated_field = (field_labels[field_name] ==
|
||||
protobuf.descriptor.FieldDescriptor.LABEL_REPEATED)
|
||||
|
||||
if sub_path:
|
||||
field = getattr(message, field_name)
|
||||
if repeated_field:
|
||||
if not field:
|
||||
return False
|
||||
return all(_HasField(entry, sub_path) for entry in field)
|
||||
else:
|
||||
return _HasField(field, sub_path)
|
||||
else:
|
||||
if repeated_field:
|
||||
return len(getattr(message, field_name)) > 0
|
||||
else:
|
||||
return message.HasField(field_name)
|
||||
|
||||
|
||||
def IsValid(cq_config):
|
||||
"""Validates a CQ config and prints errors/warnings to the screen.
|
||||
|
||||
Args:
|
||||
cq_config (string): Unparsed text format of the CQ config proto.
|
||||
|
||||
Returns:
|
||||
True if the config is valid.
|
||||
"""
|
||||
try:
|
||||
config = cq_pb2.Config()
|
||||
protobuf.text_format.Merge(cq_config, config)
|
||||
except protobuf.text_format.ParseError as e:
|
||||
logging.error('Failed to parse config as protobuf:\n%s', e)
|
||||
return False
|
||||
|
||||
if _HasField(config, 'gerrit'):
|
||||
if _HasField(config, 'rietveld'):
|
||||
logging.error('gerrit and rietveld are not supported at the same time.')
|
||||
return False
|
||||
# TODO(tandrii): validate gerrit.
|
||||
required_fields = REQUIRED_FIELDS + ['gerrit.cq_verified_label']
|
||||
if _HasField(config, 'verifiers.reviewer_lgtm'):
|
||||
logging.error('reviewer_lgtm verifier is not supported with Gerrit.')
|
||||
return False
|
||||
elif _HasField(config, 'rietveld'):
|
||||
required_fields = REQUIRED_FIELDS + ['rietveld.url']
|
||||
else:
|
||||
logging.error('either rietveld gerrit are required fields.')
|
||||
return False
|
||||
|
||||
for fname in required_fields:
|
||||
if not _HasField(config, fname):
|
||||
logging.error('%s is a required field', fname)
|
||||
return False
|
||||
|
||||
for fname in LEGACY_FIELDS:
|
||||
if _HasField(config, fname):
|
||||
logging.warn('%s is a legacy field', fname)
|
||||
|
||||
|
||||
for base in config.rietveld.project_bases:
|
||||
try:
|
||||
re.compile(base)
|
||||
except re.error:
|
||||
logging.error('failed to parse "%s" in project_bases as a regexp', base)
|
||||
return False
|
||||
|
||||
# TODO(sergiyb): For each field, check valid values depending on its
|
||||
# semantics, e.g. email addresses, regular expressions etc.
|
||||
return True
|
||||
Loading…
Reference in New Issue