mirror of https://github.com/OISF/suricata
prscript: remove as it is obsolete
Victor Julien
2 years ago
parent
aee41957e1
commit
9520aac79c
@ -1,421 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright(C) 2013-2020 Open Information Security Foundation
|
||||
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, version 2 of the License.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
|
||||
# Note to Docker users:
|
||||
# If you are running SELinux in enforced mode, you may want to run
|
||||
# chcon -Rt svirt_sandbox_file_t SURICATA_ROOTSRC_DIR
|
||||
# or the buildbot will not be able to access to the data in /data/oisf
|
||||
# and the git step will fail.
|
||||
|
||||
import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, http.cookiejar
|
||||
try:
|
||||
import simplejson as json
|
||||
except:
|
||||
import json
|
||||
import time
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import copy
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
GOT_NOTIFY = True
|
||||
try:
|
||||
import pynotify
|
||||
except:
|
||||
GOT_NOTIFY = False
|
||||
|
||||
GOT_DOCKER = True
|
||||
GOT_DOCKERPY_API = 2
|
||||
|
||||
try:
|
||||
import docker
|
||||
try:
|
||||
from docker import APIClient as Client
|
||||
from docker import from_env as DockerClient
|
||||
except ImportError:
|
||||
GOT_DOCKERPY_API = 1
|
||||
try:
|
||||
from docker import Client
|
||||
except ImportError:
|
||||
GOT_DOCKER = False
|
||||
except ImportError:
|
||||
GOT_DOCKER = False
|
||||
|
||||
# variables
|
||||
# - github user
|
||||
# - buildbot user and password
|
||||
|
||||
BASE_URI="https://buildbot.openinfosecfoundation.org/"
|
||||
GITHUB_BASE_URI = "https://api.github.com/repos/"
|
||||
GITHUB_MASTER_URI = "https://api.github.com/repos/OISF/suricata/commits?sha=master"
|
||||
|
||||
if GOT_DOCKER:
|
||||
parser = argparse.ArgumentParser(prog='prscript', description='Script checking validity of branch before PR')
|
||||
else:
|
||||
parser = argparse.ArgumentParser(prog='prscript', description='Script checking validity of branch before PR',
|
||||
epilog='You need to install Python docker module to enable docker container handling options.')
|
||||
parser.add_argument('-u', '--username', dest='username', help='github and buildbot user')
|
||||
parser.add_argument('-p', '--password', dest='password', help='buildbot password')
|
||||
parser.add_argument('-c', '--check', action='store_const', const=True, help='only check last build', default=False)
|
||||
parser.add_argument('-v', '--verbose', action='store_const', const=True, help='verbose output', default=False)
|
||||
parser.add_argument('--norebase', action='store_const', const=True, help='do not test if branch is in sync with master', default=False)
|
||||
parser.add_argument('-r', '--repository', dest='repository', default='suricata', help='name of suricata repository on github')
|
||||
parser.add_argument('-l', '--local', action='store_const', const=True, help='local testing before github push', default=False)
|
||||
if GOT_NOTIFY:
|
||||
parser.add_argument('-n', '--notify', action='store_const', const=True, help='send desktop notification', default=False)
|
||||
|
||||
docker_deps = ""
|
||||
if not GOT_DOCKER:
|
||||
docker_deps = " (disabled)"
|
||||
parser.add_argument('-d', '--docker', action='store_const', const=True, help='use docker based testing', default=False)
|
||||
parser.add_argument('-C', '--create', action='store_const', const=True, help='create docker container' + docker_deps, default=False)
|
||||
parser.add_argument('-s', '--start', action='store_const', const=True, help='start docker container' + docker_deps, default=False)
|
||||
parser.add_argument('-S', '--stop', action='store_const', const=True, help='stop docker container' + docker_deps, default=False)
|
||||
parser.add_argument('-R', '--rm', action='store_const', const=True, help='remove docker container and image' + docker_deps, default=False)
|
||||
parser.add_argument('branch', metavar='branch', help='github branch to build', nargs='?')
|
||||
args = parser.parse_args()
|
||||
username = args.username
|
||||
password = args.password
|
||||
cookie = None
|
||||
|
||||
if args.create or args.start or args.stop or args.rm:
|
||||
if GOT_DOCKER:
|
||||
args.docker = True
|
||||
args.local = True
|
||||
else:
|
||||
print("You need to install python docker to use docker handling features.")
|
||||
sys.exit(-1)
|
||||
|
||||
if not args.local:
|
||||
if not args.username:
|
||||
print("You need to specify a github username (-u option) for this mode (or use -l to disable)")
|
||||
sys.exit(-1)
|
||||
|
||||
if args.docker:
|
||||
BASE_URI="http://localhost:8010/"
|
||||
BUILDERS_LIST = ["gcc", "clang", "debug", "features", "profiling", "pcaps"]
|
||||
else:
|
||||
BUILDERS_LIST = [username, username + "-pcap"]
|
||||
|
||||
BUILDERS_URI=BASE_URI+"builders/"
|
||||
JSON_BUILDERS_URI=BASE_URI+"json/builders/"
|
||||
|
||||
if GOT_NOTIFY:
|
||||
if args.notify:
|
||||
pynotify.init("PRscript")
|
||||
|
||||
def SendNotification(title, text):
|
||||
if not GOT_NOTIFY:
|
||||
return
|
||||
if not args.notify:
|
||||
return
|
||||
n = pynotify.Notification(title, text)
|
||||
n.show()
|
||||
|
||||
def TestRepoSync(branch):
|
||||
request = urllib.request.Request(GITHUB_MASTER_URI)
|
||||
page = urllib.request.urlopen(request)
|
||||
json_result = json.loads(page.read())
|
||||
sha_orig = json_result[0]["sha"]
|
||||
check_command = ["git", "branch", "--contains", sha_orig ]
|
||||
p1 = Popen(check_command, stdout=PIPE)
|
||||
p2 = Popen(["grep", branch], stdin=p1.stdout, stdout=PIPE)
|
||||
p1.stdout.close()
|
||||
output = p2.communicate()[0]
|
||||
if len(output) == 0:
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def TestGithubSync(branch):
|
||||
request = urllib.request.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1")
|
||||
try:
|
||||
page = urllib.request.urlopen(request)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return -2
|
||||
else:
|
||||
raise(e)
|
||||
json_result = json.loads(page.read())
|
||||
sha_github = json_result[0]["sha"]
|
||||
check_command = ["git", "rev-parse", branch]
|
||||
p1 = Popen(check_command, stdout=PIPE)
|
||||
sha_local = p1.communicate()[0].decode('ascii').rstrip()
|
||||
if sha_local != sha_github:
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def OpenBuildbotSession():
|
||||
auth_params = { 'username':username,'passwd':password, 'name':'login'}
|
||||
cookie = http.cookiejar.LWPCookieJar()
|
||||
params = urllib.parse.urlencode(auth_params).encode('ascii')
|
||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
|
||||
urllib.request.install_opener(opener)
|
||||
request = urllib.request.Request(BASE_URI + 'login', params)
|
||||
_ = urllib.request.urlopen(request)
|
||||
return cookie
|
||||
|
||||
|
||||
def SubmitBuild(branch, extension = "", builder_name = None):
|
||||
raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'}
|
||||
params = urllib.parse.urlencode(raw_params).encode('ascii')
|
||||
if not args.docker:
|
||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
|
||||
urllib.request.install_opener(opener)
|
||||
if builder_name == None:
|
||||
builder_name = username + extension
|
||||
request = urllib.request.Request(BUILDERS_URI + builder_name + '/force', params)
|
||||
page = urllib.request.urlopen(request)
|
||||
|
||||
result = page.read()
|
||||
if args.verbose:
|
||||
print("=== response ===")
|
||||
print(result)
|
||||
print("=== end of response ===")
|
||||
if args.docker:
|
||||
if "<h2>Pending Build Requests:</h2>" in result:
|
||||
print("Build '" + builder_name + "' submitted")
|
||||
return 0
|
||||
else:
|
||||
return -1
|
||||
if b'Current Builds' in result:
|
||||
print("Build '" + builder_name + "' submitted")
|
||||
return 0
|
||||
else:
|
||||
return -1
|
||||
|
||||
# TODO honor the branch argument
|
||||
def FindBuild(branch, extension = "", builder_name = None):
|
||||
if builder_name == None:
|
||||
request = urllib.request.Request(JSON_BUILDERS_URI + username + extension + '/')
|
||||
else:
|
||||
request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/')
|
||||
page = urllib.request.urlopen(request)
|
||||
json_result = json.loads(page.read())
|
||||
# Pending build is unnumbered
|
||||
if json_result["pendingBuilds"]:
|
||||
return -1
|
||||
if json_result["currentBuilds"]:
|
||||
return json_result["currentBuilds"][0]
|
||||
if json_result["cachedBuilds"]:
|
||||
return json_result["cachedBuilds"][-1]
|
||||
return -2
|
||||
|
||||
def GetBuildStatus(builder, buildid, extension="", builder_name = None):
|
||||
if builder_name == None:
|
||||
builder_name = username + extension
|
||||
# https://buildbot.oisf.net/json/builders/build%20deb6/builds/11
|
||||
request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid))
|
||||
page = urllib.request.urlopen(request)
|
||||
result = page.read()
|
||||
if args.verbose:
|
||||
print("=== response ===")
|
||||
print(result)
|
||||
print("=== end of response ===")
|
||||
json_result = json.loads(result)
|
||||
if json_result["currentStep"]:
|
||||
return 1
|
||||
if 'successful' in json_result["text"]:
|
||||
return 0
|
||||
return -1
|
||||
|
||||
def WaitForBuildResult(builder, buildid, extension="", builder_name = None):
|
||||
# fetch result every 10 secs till task is over
|
||||
if builder_name == None:
|
||||
builder_name = username + extension
|
||||
res = 1
|
||||
while res == 1:
|
||||
res = GetBuildStatus(username,buildid, builder_name = builder_name)
|
||||
if res == 1:
|
||||
time.sleep(10)
|
||||
|
||||
# return the result
|
||||
if res == 0:
|
||||
print("Build successful for " + builder_name)
|
||||
else:
|
||||
print("Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid))
|
||||
return res
|
||||
|
||||
# check that github branch and OISF master branch are sync
|
||||
if not args.local:
|
||||
ret = TestGithubSync(args.branch)
|
||||
if ret != 0:
|
||||
if ret == -2:
|
||||
print("Branch " + args.branch + " is not pushed to Github.")
|
||||
sys.exit(-1)
|
||||
if args.norebase:
|
||||
print("Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option.")
|
||||
else:
|
||||
print("Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed.")
|
||||
sys.exit(-1)
|
||||
if TestRepoSync(args.branch) != 0:
|
||||
if args.norebase:
|
||||
print("Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option.")
|
||||
else:
|
||||
print("Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed.")
|
||||
sys.exit(-1)
|
||||
|
||||
def CreateContainer():
|
||||
# FIXME check if existing
|
||||
print("Pulling docking image, first run should take long")
|
||||
if GOT_DOCKERPY_API < 2:
|
||||
cli = Client()
|
||||
cli.pull('regit/suri-buildbot')
|
||||
cli.create_container(name='suri-buildbot', image='regit/suri-buildbot', ports=[8010, 22], volumes=['/data/oisf', '/data/buildbot/master/master.cfg'])
|
||||
else:
|
||||
cli = DockerClient()
|
||||
cli.images.pull('regit/suri-buildbot')
|
||||
suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
|
||||
print("Using base src dir: " + suri_src_dir)
|
||||
cli.containers.create('regit/suri-buildbot', name='suri-buildbot', ports={'8010/tcp': 8010, '22/tcp': None} , volumes={suri_src_dir: { 'bind': '/data/oisf', 'mode': 'ro'}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'mode': 'ro'}}, detach = True)
|
||||
sys.exit(0)
|
||||
|
||||
def StartContainer():
|
||||
suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
|
||||
print("Using base src dir: " + suri_src_dir)
|
||||
if GOT_DOCKERPY_API < 2:
|
||||
cli = Client()
|
||||
cli.start('suri-buildbot', port_bindings={8010:8010, 22:None}, binds={suri_src_dir: { 'bind': '/data/oisf', 'ro': True}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'ro': True}} )
|
||||
else:
|
||||
cli = DockerClient()
|
||||
cli.containers.get('suri-buildbot').start()
|
||||
sys.exit(0)
|
||||
|
||||
def StopContainer():
|
||||
if GOT_DOCKERPY_API < 2:
|
||||
cli = Client()
|
||||
cli.stop('suri-buildbot')
|
||||
else:
|
||||
cli = DockerClient()
|
||||
cli.containers.get('suri-buildbot').stop()
|
||||
sys.exit(0)
|
||||
|
||||
def RmContainer():
|
||||
if GOT_DOCKERPY_API < 2:
|
||||
cli = Client()
|
||||
try:
|
||||
cli.remove_container('suri-buildbot')
|
||||
except:
|
||||
print("Unable to remove suri-buildbot container")
|
||||
try:
|
||||
cli.remove_image('regit/suri-buildbot:latest')
|
||||
except:
|
||||
print("Unable to remove suri-buildbot images")
|
||||
else:
|
||||
cli = DockerClient()
|
||||
cli.containers.get('suri-buildbot').remove()
|
||||
cli.images.remove('regit/suri-buildbot:latest')
|
||||
sys.exit(0)
|
||||
|
||||
if GOT_DOCKER:
|
||||
if args.create:
|
||||
CreateContainer()
|
||||
if args.start:
|
||||
StartContainer()
|
||||
if args.stop:
|
||||
StopContainer()
|
||||
if args.rm:
|
||||
RmContainer()
|
||||
|
||||
if not args.branch:
|
||||
print("You need to specify a branch for this mode")
|
||||
sys.exit(-1)
|
||||
|
||||
# submit buildbot form to build current branch on the devel builder
|
||||
if not args.check:
|
||||
if not args.docker:
|
||||
cookie = OpenBuildbotSession()
|
||||
if cookie == None:
|
||||
print("Unable to connect to buildbot with provided credentials")
|
||||
sys.exit(-1)
|
||||
for build in BUILDERS_LIST:
|
||||
res = SubmitBuild(args.branch, builder_name = build)
|
||||
if res == -1:
|
||||
print("Unable to start build. Check command line parameters")
|
||||
sys.exit(-1)
|
||||
|
||||
buildids = {}
|
||||
|
||||
if args.docker:
|
||||
time.sleep(2)
|
||||
|
||||
# get build number and exit if we don't have
|
||||
for build in BUILDERS_LIST:
|
||||
buildid = FindBuild(args.branch, builder_name = build)
|
||||
if buildid == -1:
|
||||
print("Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build)
|
||||
elif buildid == -2:
|
||||
print("No build found for " + BUILDERS_URI + build)
|
||||
sys.exit(0)
|
||||
else:
|
||||
if not args.docker:
|
||||
print("You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid))
|
||||
buildids[build] = buildid
|
||||
|
||||
if args.docker:
|
||||
print("You can watch build progress at " + BASE_URI + "waterfall")
|
||||
|
||||
if len(buildids):
|
||||
print("Waiting for build completion")
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
buildres = 0
|
||||
if args.docker:
|
||||
while len(buildids):
|
||||
up_buildids = copy.copy(buildids)
|
||||
for build in buildids:
|
||||
ret = GetBuildStatus(build, buildids[build], builder_name = build)
|
||||
if ret == -1:
|
||||
buildres = -1
|
||||
up_buildids.pop(build, None)
|
||||
if len(up_buildids):
|
||||
remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")"
|
||||
else:
|
||||
remains = ""
|
||||
print("Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains)
|
||||
elif ret == 0:
|
||||
up_buildids.pop(build, None)
|
||||
if len(up_buildids):
|
||||
remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")"
|
||||
else:
|
||||
remains = ""
|
||||
print("Build successful for " + build + remains)
|
||||
time.sleep(5)
|
||||
buildids = up_buildids
|
||||
if res == -1:
|
||||
SendNotification("PRscript failure", "Some builds have failed. Check <a href='" + BASE_URI + "waterfall'>waterfall</a> for results.")
|
||||
sys.exit(-1)
|
||||
else:
|
||||
print("PRscript completed successfully")
|
||||
SendNotification("PRscript success", "Congrats! All builds have passed.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
for build in buildids:
|
||||
res = WaitForBuildResult(build, buildids[build], builder_name = build)
|
||||
if res == -1:
|
||||
buildres = -1
|
||||
|
||||
if buildres == 0:
|
||||
if not args.norebase and not args.docker:
|
||||
print("You can copy/paste following lines into github PR")
|
||||
for build in buildids:
|
||||
print("- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build]))
|
||||
SendNotification("OISF PRscript success", "Congrats! All builds have passed.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
SendNotification("OISF PRscript failure", "Some builds have failed. Check <a href='" + BASE_URI + "waterfall'>waterfall</a> for results.")
|
||||
sys.exit(-1)
|
||||
Loading…
Reference in New Issue