diff --git a/qa/prscript.py b/qa/prscript.py deleted file mode 100755 index 10c090f549..0000000000 --- a/qa/prscript.py +++ /dev/null @@ -1,421 +0,0 @@ -#!/usr/bin/env python3 -# Copyright(C) 2013-2020 Open Information Security Foundation - -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -# Note to Docker users: -# If you are running SELinux in enforced mode, you may want to run -# chcon -Rt svirt_sandbox_file_t SURICATA_ROOTSRC_DIR -# or the buildbot will not be able to access to the data in /data/oisf -# and the git step will fail. - -import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, http.cookiejar -try: - import simplejson as json -except: - import json -import time -import argparse -import sys -import os -import copy -from subprocess import Popen, PIPE - -GOT_NOTIFY = True -try: - import pynotify -except: - GOT_NOTIFY = False - -GOT_DOCKER = True -GOT_DOCKERPY_API = 2 - -try: - import docker - try: - from docker import APIClient as Client - from docker import from_env as DockerClient - except ImportError: - GOT_DOCKERPY_API = 1 - try: - from docker import Client - except ImportError: - GOT_DOCKER = False -except ImportError: - GOT_DOCKER = False - -# variables -# - github user -# - buildbot user and password - -BASE_URI="https://buildbot.openinfosecfoundation.org/" -GITHUB_BASE_URI = "https://api.github.com/repos/" -GITHUB_MASTER_URI = "https://api.github.com/repos/OISF/suricata/commits?sha=master" - -if GOT_DOCKER: - parser = argparse.ArgumentParser(prog='prscript', description='Script checking validity of branch before PR') -else: - parser = argparse.ArgumentParser(prog='prscript', description='Script checking validity of branch before PR', - epilog='You need to install Python docker module to enable docker container handling options.') -parser.add_argument('-u', '--username', dest='username', help='github and buildbot user') -parser.add_argument('-p', '--password', dest='password', help='buildbot password') -parser.add_argument('-c', '--check', action='store_const', const=True, help='only check last build', default=False) -parser.add_argument('-v', '--verbose', action='store_const', const=True, help='verbose output', default=False) -parser.add_argument('--norebase', action='store_const', const=True, help='do not test if branch is in sync with master', default=False) -parser.add_argument('-r', '--repository', dest='repository', default='suricata', help='name of suricata repository on github') -parser.add_argument('-l', '--local', action='store_const', const=True, help='local testing before github push', default=False) -if GOT_NOTIFY: - parser.add_argument('-n', '--notify', action='store_const', const=True, help='send desktop notification', default=False) - -docker_deps = "" -if not GOT_DOCKER: - docker_deps = " (disabled)" -parser.add_argument('-d', '--docker', action='store_const', const=True, help='use docker based testing', default=False) -parser.add_argument('-C', '--create', action='store_const', const=True, help='create docker container' + docker_deps, default=False) -parser.add_argument('-s', '--start', action='store_const', const=True, help='start docker container' + docker_deps, default=False) -parser.add_argument('-S', '--stop', action='store_const', const=True, help='stop docker container' + docker_deps, default=False) -parser.add_argument('-R', '--rm', action='store_const', const=True, help='remove docker container and image' + docker_deps, default=False) -parser.add_argument('branch', metavar='branch', help='github branch to build', nargs='?') -args = parser.parse_args() -username = args.username -password = args.password -cookie = None - -if args.create or args.start or args.stop or args.rm: - if GOT_DOCKER: - args.docker = True - args.local = True - else: - print("You need to install python docker to use docker handling features.") - sys.exit(-1) - -if not args.local: - if not args.username: - print("You need to specify a github username (-u option) for this mode (or use -l to disable)") - sys.exit(-1) - -if args.docker: - BASE_URI="http://localhost:8010/" - BUILDERS_LIST = ["gcc", "clang", "debug", "features", "profiling", "pcaps"] -else: - BUILDERS_LIST = [username, username + "-pcap"] - -BUILDERS_URI=BASE_URI+"builders/" -JSON_BUILDERS_URI=BASE_URI+"json/builders/" - -if GOT_NOTIFY: - if args.notify: - pynotify.init("PRscript") - -def SendNotification(title, text): - if not GOT_NOTIFY: - return - if not args.notify: - return - n = pynotify.Notification(title, text) - n.show() - -def TestRepoSync(branch): - request = urllib.request.Request(GITHUB_MASTER_URI) - page = urllib.request.urlopen(request) - json_result = json.loads(page.read()) - sha_orig = json_result[0]["sha"] - check_command = ["git", "branch", "--contains", sha_orig ] - p1 = Popen(check_command, stdout=PIPE) - p2 = Popen(["grep", branch], stdin=p1.stdout, stdout=PIPE) - p1.stdout.close() - output = p2.communicate()[0] - if len(output) == 0: - return -1 - return 0 - -def TestGithubSync(branch): - request = urllib.request.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1") - try: - page = urllib.request.urlopen(request) - except urllib.error.HTTPError as e: - if e.code == 404: - return -2 - else: - raise(e) - json_result = json.loads(page.read()) - sha_github = json_result[0]["sha"] - check_command = ["git", "rev-parse", branch] - p1 = Popen(check_command, stdout=PIPE) - sha_local = p1.communicate()[0].decode('ascii').rstrip() - if sha_local != sha_github: - return -1 - return 0 - -def OpenBuildbotSession(): - auth_params = { 'username':username,'passwd':password, 'name':'login'} - cookie = http.cookiejar.LWPCookieJar() - params = urllib.parse.urlencode(auth_params).encode('ascii') - opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie)) - urllib.request.install_opener(opener) - request = urllib.request.Request(BASE_URI + 'login', params) - _ = urllib.request.urlopen(request) - return cookie - - -def SubmitBuild(branch, extension = "", builder_name = None): - raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'} - params = urllib.parse.urlencode(raw_params).encode('ascii') - if not args.docker: - opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie)) - urllib.request.install_opener(opener) - if builder_name == None: - builder_name = username + extension - request = urllib.request.Request(BUILDERS_URI + builder_name + '/force', params) - page = urllib.request.urlopen(request) - - result = page.read() - if args.verbose: - print("=== response ===") - print(result) - print("=== end of response ===") - if args.docker: - if "

Pending Build Requests:

" in result: - print("Build '" + builder_name + "' submitted") - return 0 - else: - return -1 - if b'Current Builds' in result: - print("Build '" + builder_name + "' submitted") - return 0 - else: - return -1 - -# TODO honor the branch argument -def FindBuild(branch, extension = "", builder_name = None): - if builder_name == None: - request = urllib.request.Request(JSON_BUILDERS_URI + username + extension + '/') - else: - request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/') - page = urllib.request.urlopen(request) - json_result = json.loads(page.read()) - # Pending build is unnumbered - if json_result["pendingBuilds"]: - return -1 - if json_result["currentBuilds"]: - return json_result["currentBuilds"][0] - if json_result["cachedBuilds"]: - return json_result["cachedBuilds"][-1] - return -2 - -def GetBuildStatus(builder, buildid, extension="", builder_name = None): - if builder_name == None: - builder_name = username + extension - # https://buildbot.oisf.net/json/builders/build%20deb6/builds/11 - request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid)) - page = urllib.request.urlopen(request) - result = page.read() - if args.verbose: - print("=== response ===") - print(result) - print("=== end of response ===") - json_result = json.loads(result) - if json_result["currentStep"]: - return 1 - if 'successful' in json_result["text"]: - return 0 - return -1 - -def WaitForBuildResult(builder, buildid, extension="", builder_name = None): - # fetch result every 10 secs till task is over - if builder_name == None: - builder_name = username + extension - res = 1 - while res == 1: - res = GetBuildStatus(username,buildid, builder_name = builder_name) - if res == 1: - time.sleep(10) - - # return the result - if res == 0: - print("Build successful for " + builder_name) - else: - print("Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid)) - return res - - # check that github branch and OISF master branch are sync -if not args.local: - ret = TestGithubSync(args.branch) - if ret != 0: - if ret == -2: - print("Branch " + args.branch + " is not pushed to Github.") - sys.exit(-1) - if args.norebase: - print("Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option.") - else: - print("Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed.") - sys.exit(-1) - if TestRepoSync(args.branch) != 0: - if args.norebase: - print("Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option.") - else: - print("Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed.") - sys.exit(-1) - -def CreateContainer(): - # FIXME check if existing - print("Pulling docking image, first run should take long") - if GOT_DOCKERPY_API < 2: - cli = Client() - cli.pull('regit/suri-buildbot') - cli.create_container(name='suri-buildbot', image='regit/suri-buildbot', ports=[8010, 22], volumes=['/data/oisf', '/data/buildbot/master/master.cfg']) - else: - cli = DockerClient() - cli.images.pull('regit/suri-buildbot') - suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - print("Using base src dir: " + suri_src_dir) - cli.containers.create('regit/suri-buildbot', name='suri-buildbot', ports={'8010/tcp': 8010, '22/tcp': None} , volumes={suri_src_dir: { 'bind': '/data/oisf', 'mode': 'ro'}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'mode': 'ro'}}, detach = True) - sys.exit(0) - -def StartContainer(): - suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - print("Using base src dir: " + suri_src_dir) - if GOT_DOCKERPY_API < 2: - cli = Client() - cli.start('suri-buildbot', port_bindings={8010:8010, 22:None}, binds={suri_src_dir: { 'bind': '/data/oisf', 'ro': True}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'ro': True}} ) - else: - cli = DockerClient() - cli.containers.get('suri-buildbot').start() - sys.exit(0) - -def StopContainer(): - if GOT_DOCKERPY_API < 2: - cli = Client() - cli.stop('suri-buildbot') - else: - cli = DockerClient() - cli.containers.get('suri-buildbot').stop() - sys.exit(0) - -def RmContainer(): - if GOT_DOCKERPY_API < 2: - cli = Client() - try: - cli.remove_container('suri-buildbot') - except: - print("Unable to remove suri-buildbot container") - try: - cli.remove_image('regit/suri-buildbot:latest') - except: - print("Unable to remove suri-buildbot images") - else: - cli = DockerClient() - cli.containers.get('suri-buildbot').remove() - cli.images.remove('regit/suri-buildbot:latest') - sys.exit(0) - -if GOT_DOCKER: - if args.create: - CreateContainer() - if args.start: - StartContainer() - if args.stop: - StopContainer() - if args.rm: - RmContainer() - -if not args.branch: - print("You need to specify a branch for this mode") - sys.exit(-1) - -# submit buildbot form to build current branch on the devel builder -if not args.check: - if not args.docker: - cookie = OpenBuildbotSession() - if cookie == None: - print("Unable to connect to buildbot with provided credentials") - sys.exit(-1) - for build in BUILDERS_LIST: - res = SubmitBuild(args.branch, builder_name = build) - if res == -1: - print("Unable to start build. Check command line parameters") - sys.exit(-1) - -buildids = {} - -if args.docker: - time.sleep(2) - -# get build number and exit if we don't have -for build in BUILDERS_LIST: - buildid = FindBuild(args.branch, builder_name = build) - if buildid == -1: - print("Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build) - elif buildid == -2: - print("No build found for " + BUILDERS_URI + build) - sys.exit(0) - else: - if not args.docker: - print("You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid)) - buildids[build] = buildid - -if args.docker: - print("You can watch build progress at " + BASE_URI + "waterfall") - -if len(buildids): - print("Waiting for build completion") -else: - sys.exit(0) - -buildres = 0 -if args.docker: - while len(buildids): - up_buildids = copy.copy(buildids) - for build in buildids: - ret = GetBuildStatus(build, buildids[build], builder_name = build) - if ret == -1: - buildres = -1 - up_buildids.pop(build, None) - if len(up_buildids): - remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")" - else: - remains = "" - print("Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains) - elif ret == 0: - up_buildids.pop(build, None) - if len(up_buildids): - remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")" - else: - remains = "" - print("Build successful for " + build + remains) - time.sleep(5) - buildids = up_buildids - if res == -1: - SendNotification("PRscript failure", "Some builds have failed. Check waterfall for results.") - sys.exit(-1) - else: - print("PRscript completed successfully") - SendNotification("PRscript success", "Congrats! All builds have passed.") - sys.exit(0) -else: - for build in buildids: - res = WaitForBuildResult(build, buildids[build], builder_name = build) - if res == -1: - buildres = -1 - -if buildres == 0: - if not args.norebase and not args.docker: - print("You can copy/paste following lines into github PR") - for build in buildids: - print("- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build])) - SendNotification("OISF PRscript success", "Congrats! All builds have passed.") - sys.exit(0) -else: - SendNotification("OISF PRscript failure", "Some builds have failed. Check waterfall for results.") - sys.exit(-1)