aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

prscript: port to python 3

Browse Source
pull/5381/head
Eric Leblond 5 years ago committed by Victor Julien
parent 8cd82486e2
commit 48c30efd34
1 changed files with 62 additions and 62 deletions
Show Stats Download Patch File Download Diff File

124
qa/prscript.py
Unescape Escape View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright(C) 2013-2020 Open Information Security Foundation
# This program is free software; you can redistribute it and/or modify
@ -20,7 +20,7 @@
# or the buildbot will not be able to access to the data in /data/oisf
# and the git step will fail.
import urllib, urllib2, cookielib
import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, http.cookiejar
try:
import simplejson as json
except:
@ -97,12 +97,12 @@ if args.create or args.start or args.stop or args.rm:
args.docker = True
args.local = True
else:
print "You need to install python docker to use docker handling features."
print("You need to install python docker to use docker handling features.")
sys.exit(-1)
if not args.local:
if not args.username:
print "You need to specify a github username (-u option) for this mode (or use -l to disable)"
print("You need to specify a github username (-u option) for this mode (or use -l to disable)")
sys.exit(-1)
if args.docker:
@ -127,8 +127,8 @@ def SendNotification(title, text):
n.show()
def TestRepoSync(branch):
request = urllib2.Request(GITHUB_MASTER_URI)
page = urllib2.urlopen(request)
request = urllib.request.Request(GITHUB_MASTER_URI)
page = urllib.request.urlopen(request)
json_result = json.loads(page.read())
sha_orig = json_result[0]["sha"]
check_command = ["git", "branch", "--contains", sha_orig ]
@ -141,10 +141,10 @@ def TestRepoSync(branch):
return 0
def TestGithubSync(branch):
request = urllib2.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1")
request = urllib.request.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1")
try:
page = urllib2.urlopen(request)
except urllib2.HTTPError, e:
page = urllib.request.urlopen(request)
except urllib.error.HTTPError as e:
if e.code == 404:
return -2
else:
@ -153,46 +153,46 @@ def TestGithubSync(branch):
sha_github = json_result[0]["sha"]
check_command = ["git", "rev-parse", branch]
p1 = Popen(check_command, stdout=PIPE)
sha_local = p1.communicate()[0].rstrip()
sha_local = p1.communicate()[0].decode('ascii').rstrip()
if sha_local != sha_github:
return -1
return 0
def OpenBuildbotSession():
auth_params = { 'username':username,'passwd':password, 'name':'login'}
cookie = cookielib.LWPCookieJar()
params = urllib.urlencode(auth_params)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
request = urllib2.Request(BASE_URI + 'login', params)
_ = urllib2.urlopen(request)
cookie = http.cookiejar.LWPCookieJar()
params = urllib.parse.urlencode(auth_params).encode('ascii')
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
urllib.request.install_opener(opener)
request = urllib.request.Request(BASE_URI + 'login', params)
_ = urllib.request.urlopen(request)
return cookie
def SubmitBuild(branch, extension = "", builder_name = None):
raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'}
params = urllib.urlencode(raw_params)
params = urllib.parse.urlencode(raw_params).encode('ascii')
if not args.docker:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
urllib.request.install_opener(opener)
if builder_name == None:
builder_name = username + extension
request = urllib2.Request(BUILDERS_URI + builder_name + '/force', params)
page = urllib2.urlopen(request)
request = urllib.request.Request(BUILDERS_URI + builder_name + '/force', params)
page = urllib.request.urlopen(request)
result = page.read()
if args.verbose:
print "=== response ==="
print result
print "=== end of response ==="
print("=== response ===")
print(result)
print("=== end of response ===")
if args.docker:
if "<h2>Pending Build Requests:</h2>" in result:
print "Build '" + builder_name + "' submitted"
print("Build '" + builder_name + "' submitted")
return 0
else:
return -1
if "Current Builds" in result:
print "Build '" + builder_name + "' submitted"
if b'Current Builds' in result:
print("Build '" + builder_name + "' submitted")
return 0
else:
return -1
@ -200,10 +200,10 @@ def SubmitBuild(branch, extension = "", builder_name = None):
# TODO honor the branch argument
def FindBuild(branch, extension = "", builder_name = None):
if builder_name == None:
request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/')
request = urllib.request.Request(JSON_BUILDERS_URI + username + extension + '/')
else:
request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/')
page = urllib2.urlopen(request)
request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/')
page = urllib.request.urlopen(request)
json_result = json.loads(page.read())
# Pending build is unnumbered
if json_result["pendingBuilds"]:
@ -218,13 +218,13 @@ def GetBuildStatus(builder, buildid, extension="", builder_name = None):
if builder_name == None:
builder_name = username + extension
# https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11
request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid))
page = urllib2.urlopen(request)
request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid))
page = urllib.request.urlopen(request)
result = page.read()
if args.verbose:
print "=== response ==="
print result
print "=== end of response ==="
print("=== response ===")
print(result)
print("=== end of response ===")
json_result = json.loads(result)
if json_result["currentStep"]:
return 1
@ -244,9 +244,9 @@ def WaitForBuildResult(builder, buildid, extension="", builder_name = None):
# return the result
if res == 0:
print "Build successful for " + builder_name
print("Build successful for " + builder_name)
else:
print "Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid)
print("Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid))
return res
# check that github branch and OISF master branch are sync
@ -254,23 +254,23 @@ if not args.local:
ret = TestGithubSync(args.branch)
if ret != 0:
if ret == -2:
print "Branch " + args.branch + " is not pushed to Github."
print("Branch " + args.branch + " is not pushed to Github.")
sys.exit(-1)
if args.norebase:
print "Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option."
print("Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option.")
else:
print "Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed."
print("Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed.")
sys.exit(-1)
if TestRepoSync(args.branch) != 0:
if args.norebase:
print "Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option."
print("Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option.")
else:
print "Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed."
print("Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed.")
sys.exit(-1)
def CreateContainer():
# FIXME check if existing
print "Pulling docking image, first run should take long"
print("Pulling docking image, first run should take long")
if GOT_DOCKERPY_API < 2:
cli = Client()
cli.pull('regit/suri-buildbot')
@ -279,13 +279,13 @@ def CreateContainer():
cli = DockerClient()
cli.images.pull('regit/suri-buildbot')
suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
print "Using base src dir: " + suri_src_dir
print("Using base src dir: " + suri_src_dir)
cli.containers.create('regit/suri-buildbot', name='suri-buildbot', ports={'8010/tcp': 8010, '22/tcp': None} , volumes={suri_src_dir: { 'bind': '/data/oisf', 'mode': 'ro'}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'mode': 'ro'}}, detach = True)
sys.exit(0)
def StartContainer():
suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
print "Using base src dir: " + suri_src_dir
print("Using base src dir: " + suri_src_dir)
if GOT_DOCKERPY_API < 2:
cli = Client()
cli.start('suri-buildbot', port_bindings={8010:8010, 22:None}, binds={suri_src_dir: { 'bind': '/data/oisf', 'ro': True}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'ro': True}} )
@ -309,11 +309,11 @@ def RmContainer():
try:
cli.remove_container('suri-buildbot')
except:
print "Unable to remove suri-buildbot container"
print("Unable to remove suri-buildbot container")
try:
cli.remove_image('regit/suri-buildbot:latest')
except:
print "Unable to remove suri-buildbot images"
print("Unable to remove suri-buildbot images")
else:
cli = DockerClient()
cli.containers.get('suri-buildbot').remove()
@ -331,7 +331,7 @@ if GOT_DOCKER:
RmContainer()
if not args.branch:
print "You need to specify a branch for this mode"
print("You need to specify a branch for this mode")
sys.exit(-1)
# submit buildbot form to build current branch on the devel builder
@ -339,12 +339,12 @@ if not args.check:
if not args.docker:
cookie = OpenBuildbotSession()
if cookie == None:
print "Unable to connect to buildbot with provided credentials"
print("Unable to connect to buildbot with provided credentials")
sys.exit(-1)
for build in BUILDERS_LIST:
res = SubmitBuild(args.branch, builder_name = build)
if res == -1:
print "Unable to start build. Check command line parameters"
print("Unable to start build. Check command line parameters")
sys.exit(-1)
buildids = {}
@ -356,20 +356,20 @@ if args.docker:
for build in BUILDERS_LIST:
buildid = FindBuild(args.branch, builder_name = build)
if buildid == -1:
print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build
print("Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build)
elif buildid == -2:
print "No build found for " + BUILDERS_URI + build
print("No build found for " + BUILDERS_URI + build)
sys.exit(0)
else:
if not args.docker:
print "You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid)
print("You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid))
buildids[build] = buildid
if args.docker:
print "You can watch build progress at " + BASE_URI + "waterfall"
print("You can watch build progress at " + BASE_URI + "waterfall")
if len(buildids):
print "Waiting for build completion"
print("Waiting for build completion")
else:
sys.exit(0)
@ -383,24 +383,24 @@ if args.docker:
buildres = -1
up_buildids.pop(build, None)
if len(up_buildids):
remains = " (remaining builds: " + ', '.join(up_buildids.keys()) + ")"
remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")"
else:
remains = ""
print "Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains
print("Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains)
elif ret == 0:
up_buildids.pop(build, None)
if len(up_buildids):
remains = " (remaining builds: " + ', '.join(up_buildids.keys()) + ")"
remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")"
else:
remains = ""
print "Build successful for " + build + remains
print("Build successful for " + build + remains)
time.sleep(5)
buildids = up_buildids
if res == -1:
SendNotification("PRscript failure", "Some builds have failed. Check <a href='" + BASE_URI + "waterfall'>waterfall</a> for results.")
sys.exit(-1)
else:
print "PRscript completed successfully"
print("PRscript completed successfully")
SendNotification("PRscript success", "Congrats! All builds have passed.")
sys.exit(0)
else:
@ -411,9 +411,9 @@ else:
if buildres == 0:
if not args.norebase and not args.docker:
print "You can copy/paste following lines into github PR"
print("You can copy/paste following lines into github PR")
for build in buildids:
print "- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build])
print("- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build]))
SendNotification("OISF PRscript success", "Congrats! All builds have passed.")
sys.exit(0)
else:

Write Preview
Loading…
Cancel
Save