Roll cq_client.
From infra_internal, revisions 6f979a..cd0150f. TBR=sergiyb@chromium.org,machenbach@chromium.org BUG= Review URL: https://codereview.chromium.org/1697053003 git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@298789 0039d316-1c4b-4281-b951-d872f2087c98changes/01/332501/1
tandrii@chromium.org
9 years ago
parent
a18cfe6573
commit
a1afbb986d
@ -1,29 +1,55 @@
|
|||||||
|
# CQ Client Library.
|
||||||
|
|
||||||
This directory contains CQ client library to be distributed to other repos. If
|
This directory contains CQ client library to be distributed to other repos. If
|
||||||
you need to modify some files in this directory, please make sure that you are
|
you need to modify some files in this directory, please make sure that you are
|
||||||
changing the canonical version of the source code and not one of the copies,
|
changing the canonical version of the source code and not one of the copies,
|
||||||
which should only be updated as a whole using Glyco (when available, see
|
which should only be updated as a whole using Glyco (when available, see
|
||||||
[chromium issue 489420](http://crbug.com/489420)).
|
[chromium issue 489420](http://crbug.com/489420)).
|
||||||
|
|
||||||
The canonical version is located at `https://chrome-internal.googlesource.com/
|
The canonical version is located at
|
||||||
infra/infra_internal/+/master/commit_queue/cq_client`.
|
[https://chrome-internal.googlesource.com/infra/infra_internal/+/master/commit_queue/cq_client]().
|
||||||
|
|
||||||
|
When modifying cq.proto, consider adding checks to validator in
|
||||||
|
[https://chrome-internal.googlesource.com/infra/infra_internal/+/master/appengine/commit_queue/src/commitqueue/validate.go]().
|
||||||
|
|
||||||
|
|
||||||
|
## Generation of Python and Go bindings
|
||||||
|
|
||||||
|
### tl;dr
|
||||||
|
|
||||||
|
make
|
||||||
|
|
||||||
|
|
||||||
|
### Details
|
||||||
|
|
||||||
|
All commands below assume you are working in a standard infra_internal gclient
|
||||||
|
checkout (e.g., after you ran `mkdir src && cd src && fetch infra_internal`) and
|
||||||
|
are in current directory of this README.md (that is, in
|
||||||
|
`cd infra_internal/commit_queue/cq_client`).
|
||||||
|
|
||||||
You'll need to use protoc version 2.6.1 and
|
To generate Python's `cq_pb2.py` you'll need to get and `protoc` of version
|
||||||
recent golang/protobuf package. Sadly, the latter has neither tags nor versions.
|
**2.6.1**. You can get it by `make py-prepare`.
|
||||||
|
|
||||||
|
make py
|
||||||
|
|
||||||
You can get protobuf by downloading archive from
|
To generate Golang's protobuf file `cq.pb.go`, you'll need to bootstrap
|
||||||
https://github.com/google/protobuf/tree/v2.6.1 and manually building it. As for
|
infra/infra repository and go utilities `make go-prepare`.
|
||||||
golang compiler, if you have go configured, just
|
|
||||||
|
|
||||||
go get -u github.com/golang/protobuf/{proto,protoc-gen-go}
|
make go
|
||||||
|
|
||||||
TODO(tandrii,sergiyb): decide how to pin the go protobuf generator.
|
## Notes
|
||||||
|
|
||||||
To generate `cq_pb2.py` and `cq.pb.go`:
|
1. Please make sure to use proto3-compatible yntax, e.g. no default
|
||||||
|
values, no required fields. As of this writing (Jan 2016),
|
||||||
|
the Go protobuf compiler has been upgraded to 3.0.0. So, if you can generate go
|
||||||
|
bindings, all is good.
|
||||||
|
|
||||||
cd commit_queue/cq_client
|
2. If after generating Python binding, CQ tests fail with:
|
||||||
protoc cq.proto --python_out $(pwd) --go_out $(pwd)
|
|
||||||
|
|
||||||
Additionally, please make sure to use proto3-compatible syntax, e.g. no default
|
TypeError: __init__() got an unexpected keyword argument 'syntax'
|
||||||
values, no required fields. Ideally, we should use proto3 generator already,
|
|
||||||
however alpha version thereof is still unstable.
|
|
||||||
|
|
||||||
|
You've probably used 3.0.0 protoc generator. We should eventually switch to 3x
|
||||||
|
Python version as well, but it requires upgrading infra's Python ENV to newer
|
||||||
|
package. See [bootstrap/README.md](../../bootstrap/README.md) for more
|
||||||
|
information. We may end up deprecating Python before all infra's Python code
|
||||||
|
can be moved to protobuf v3.
|
||||||
|
|||||||
@ -1,2 +0,0 @@
|
|||||||
|
|
||||||
|
|
||||||
@ -1,66 +0,0 @@
|
|||||||
# Copyright 2015 The Chromium Authors. All rights reserved.
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
|
|
||||||
"""Unit tests for tools/validate_config.py."""
|
|
||||||
|
|
||||||
import mock
|
|
||||||
import os
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
from cq_client import cq_pb2
|
|
||||||
from cq_client import validate_config
|
|
||||||
|
|
||||||
|
|
||||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
||||||
|
|
||||||
|
|
||||||
class TestValidateConfig(unittest.TestCase):
|
|
||||||
def test_is_valid_rietveld(self):
|
|
||||||
with open(os.path.join(TEST_DIR, 'cq_rietveld.cfg'), 'r') as test_config:
|
|
||||||
self.assertTrue(validate_config.IsValid(test_config.read()))
|
|
||||||
|
|
||||||
def test_is_valid_gerrit(self):
|
|
||||||
with open(os.path.join(TEST_DIR, 'cq_gerrit.cfg'), 'r') as test_config:
|
|
||||||
self.assertTrue(validate_config.IsValid(test_config.read()))
|
|
||||||
|
|
||||||
def test_one_codereview(self):
|
|
||||||
with open(os.path.join(TEST_DIR, 'cq_gerrit.cfg'), 'r') as gerrit_config:
|
|
||||||
data = gerrit_config.read()
|
|
||||||
data += '\n'.join([
|
|
||||||
'rietveld{',
|
|
||||||
'url: "https://blabla.com"',
|
|
||||||
'}'
|
|
||||||
])
|
|
||||||
self.assertFalse(validate_config.IsValid(data))
|
|
||||||
|
|
||||||
def test_has_field(self):
|
|
||||||
config = cq_pb2.Config()
|
|
||||||
|
|
||||||
self.assertFalse(validate_config._HasField(config, 'version'))
|
|
||||||
config.version = 1
|
|
||||||
self.assertTrue(validate_config._HasField(config, 'version'))
|
|
||||||
|
|
||||||
self.assertFalse(validate_config._HasField(
|
|
||||||
config, 'rietveld.project_bases'))
|
|
||||||
config.rietveld.project_bases.append('foo://bar')
|
|
||||||
self.assertTrue(validate_config._HasField(
|
|
||||||
config, 'rietveld.project_bases'))
|
|
||||||
|
|
||||||
self.assertFalse(validate_config._HasField(
|
|
||||||
config, 'verifiers.try_job.buckets'))
|
|
||||||
self.assertFalse(validate_config._HasField(
|
|
||||||
config, 'verifiers.try_job.buckets.name'))
|
|
||||||
|
|
||||||
bucket = config.verifiers.try_job.buckets.add()
|
|
||||||
bucket.name = 'tryserver.chromium.linux'
|
|
||||||
|
|
||||||
|
|
||||||
self.assertTrue(validate_config._HasField(
|
|
||||||
config, 'verifiers.try_job.buckets'))
|
|
||||||
self.assertTrue(validate_config._HasField(
|
|
||||||
config, 'verifiers.try_job.buckets.name'))
|
|
||||||
|
|
||||||
config.verifiers.try_job.buckets.add()
|
|
||||||
self.assertFalse(validate_config._HasField(
|
|
||||||
config, 'verifiers.try_job.buckets.name'))
|
|
||||||
@ -1,125 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
# Copyright 2015 The Chromium Authors. All rights reserved.
|
|
||||||
# Use of this source code is governed by a BSD-style license that can be
|
|
||||||
# found in the LICENSE file.
|
|
||||||
|
|
||||||
"""CQ config validation library."""
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
# This file was originally copied together with the cq_client library from the
|
|
||||||
# internal commit_queue repository and then modified to import protobuf26
|
|
||||||
# instead of google.protobuf to prevent conflicts with a different version of
|
|
||||||
# google.protobuf that some users of depot_tools have installed. If you need to
|
|
||||||
# update this file, please make similar changes again and add this comment back.
|
|
||||||
# More details on why we chose to rename the package can be found in the file
|
|
||||||
# depot_tools/third_party/protobuf26/README.chromium.
|
|
||||||
import protobuf26 as protobuf
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from cq_client import cq_pb2
|
|
||||||
|
|
||||||
|
|
||||||
REQUIRED_FIELDS = [
|
|
||||||
'version',
|
|
||||||
'verifiers',
|
|
||||||
'cq_name',
|
|
||||||
]
|
|
||||||
|
|
||||||
LEGACY_FIELDS = [
|
|
||||||
'svn_repo_url',
|
|
||||||
]
|
|
||||||
|
|
||||||
EMAIL_REGEXP = '^[^@]+@[^@]+\.[^@]+$'
|
|
||||||
|
|
||||||
|
|
||||||
def _HasField(message, field_path):
|
|
||||||
"""Checks that at least one field with given path exist in the proto message.
|
|
||||||
|
|
||||||
This function correctly handles repeated fields and will make sure that each
|
|
||||||
repeated field will have required sub-path, e.g. if 'abc' is a repeated field
|
|
||||||
and field_path is 'abc.def', then the function will only return True when each
|
|
||||||
entry for 'abc' will contain at least one value for 'def'.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
message (google.protobuf.message.Message): Protocol Buffer message to check.
|
|
||||||
field_path (string): Path to the target field separated with ".".
|
|
||||||
|
|
||||||
Return:
|
|
||||||
True if at least one such field is explicitly set in the message.
|
|
||||||
"""
|
|
||||||
path_parts = field_path.split('.', 1)
|
|
||||||
field_name = path_parts[0]
|
|
||||||
sub_path = path_parts[1] if len(path_parts) == 2 else None
|
|
||||||
|
|
||||||
field_labels = {fd.name: fd.label for fd in message.DESCRIPTOR.fields}
|
|
||||||
repeated_field = (field_labels[field_name] ==
|
|
||||||
protobuf.descriptor.FieldDescriptor.LABEL_REPEATED)
|
|
||||||
|
|
||||||
if sub_path:
|
|
||||||
field = getattr(message, field_name)
|
|
||||||
if repeated_field:
|
|
||||||
if not field:
|
|
||||||
return False
|
|
||||||
return all(_HasField(entry, sub_path) for entry in field)
|
|
||||||
else:
|
|
||||||
return _HasField(field, sub_path)
|
|
||||||
else:
|
|
||||||
if repeated_field:
|
|
||||||
return len(getattr(message, field_name)) > 0
|
|
||||||
else:
|
|
||||||
return message.HasField(field_name)
|
|
||||||
|
|
||||||
|
|
||||||
def IsValid(cq_config):
|
|
||||||
"""Validates a CQ config and prints errors/warnings to the screen.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
cq_config (string): Unparsed text format of the CQ config proto.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
True if the config is valid.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
config = cq_pb2.Config()
|
|
||||||
protobuf.text_format.Merge(cq_config, config)
|
|
||||||
except protobuf.text_format.ParseError as e:
|
|
||||||
logging.error('Failed to parse config as protobuf:\n%s', e)
|
|
||||||
return False
|
|
||||||
|
|
||||||
if _HasField(config, 'gerrit'):
|
|
||||||
if _HasField(config, 'rietveld'):
|
|
||||||
logging.error('gerrit and rietveld are not supported at the same time.')
|
|
||||||
return False
|
|
||||||
# TODO(tandrii): validate gerrit.
|
|
||||||
required_fields = REQUIRED_FIELDS + ['gerrit.cq_verified_label']
|
|
||||||
if _HasField(config, 'verifiers.reviewer_lgtm'):
|
|
||||||
logging.error('reviewer_lgtm verifier is not supported with Gerrit.')
|
|
||||||
return False
|
|
||||||
elif _HasField(config, 'rietveld'):
|
|
||||||
required_fields = REQUIRED_FIELDS + ['rietveld.url']
|
|
||||||
else:
|
|
||||||
logging.error('either rietveld gerrit are required fields.')
|
|
||||||
return False
|
|
||||||
|
|
||||||
for fname in required_fields:
|
|
||||||
if not _HasField(config, fname):
|
|
||||||
logging.error('%s is a required field', fname)
|
|
||||||
return False
|
|
||||||
|
|
||||||
for fname in LEGACY_FIELDS:
|
|
||||||
if _HasField(config, fname):
|
|
||||||
logging.warn('%s is a legacy field', fname)
|
|
||||||
|
|
||||||
|
|
||||||
for base in config.rietveld.project_bases:
|
|
||||||
try:
|
|
||||||
re.compile(base)
|
|
||||||
except re.error:
|
|
||||||
logging.error('failed to parse "%s" in project_bases as a regexp', base)
|
|
||||||
return False
|
|
||||||
|
|
||||||
# TODO(sergiyb): For each field, check valid values depending on its
|
|
||||||
# semantics, e.g. email addresses, regular expressions etc.
|
|
||||||
return True
|
|
||||||
Loading…
Reference in New Issue