aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

prscript: port to python 3

Browse Source
pull/5381/head
Eric Leblond 5 years ago committed by Victor Julien
parent 8cd82486e2
commit 48c30efd34
1 changed files with 62 additions and 62 deletions
Show Stats Download Patch File Download Diff File

124
qa/prscript.py
Unescape Escape View File

@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# Copyright(C) 2013-2020 Open Information Security Foundation # Copyright(C) 2013-2020 Open Information Security Foundation
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
@ -20,7 +20,7 @@
# or the buildbot will not be able to access to the data in /data/oisf # or the buildbot will not be able to access to the data in /data/oisf
# and the git step will fail. # and the git step will fail.
import urllib, urllib2, cookielib import urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, http.cookiejar
try: try:
import simplejson as json import simplejson as json
except: except:
@ -97,12 +97,12 @@ if args.create or args.start or args.stop or args.rm:
args.docker = True args.docker = True
args.local = True args.local = True
else: else:
print "You need to install python docker to use docker handling features." print("You need to install python docker to use docker handling features.")
sys.exit(-1) sys.exit(-1)
if not args.local: if not args.local:
if not args.username: if not args.username:
print "You need to specify a github username (-u option) for this mode (or use -l to disable)" print("You need to specify a github username (-u option) for this mode (or use -l to disable)")
sys.exit(-1) sys.exit(-1)
if args.docker: if args.docker:
@ -127,8 +127,8 @@ def SendNotification(title, text):
n.show() n.show()
def TestRepoSync(branch): def TestRepoSync(branch):
request = urllib2.Request(GITHUB_MASTER_URI) request = urllib.request.Request(GITHUB_MASTER_URI)
page = urllib2.urlopen(request) page = urllib.request.urlopen(request)
json_result = json.loads(page.read()) json_result = json.loads(page.read())
sha_orig = json_result[0]["sha"] sha_orig = json_result[0]["sha"]
check_command = ["git", "branch", "--contains", sha_orig ] check_command = ["git", "branch", "--contains", sha_orig ]
@ -141,10 +141,10 @@ def TestRepoSync(branch):
return 0 return 0
def TestGithubSync(branch): def TestGithubSync(branch):
request = urllib2.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1") request = urllib.request.Request(GITHUB_BASE_URI + username + "/" + args.repository + "/commits?sha=" + branch + "&per_page=1")
try: try:
page = urllib2.urlopen(request) page = urllib.request.urlopen(request)
except urllib2.HTTPError, e: except urllib.error.HTTPError as e:
if e.code == 404: if e.code == 404:
return -2 return -2
else: else:
@ -153,46 +153,46 @@ def TestGithubSync(branch):
sha_github = json_result[0]["sha"] sha_github = json_result[0]["sha"]
check_command = ["git", "rev-parse", branch] check_command = ["git", "rev-parse", branch]
p1 = Popen(check_command, stdout=PIPE) p1 = Popen(check_command, stdout=PIPE)
sha_local = p1.communicate()[0].rstrip() sha_local = p1.communicate()[0].decode('ascii').rstrip()
if sha_local != sha_github: if sha_local != sha_github:
return -1 return -1
return 0 return 0
def OpenBuildbotSession(): def OpenBuildbotSession():
auth_params = { 'username':username,'passwd':password, 'name':'login'} auth_params = { 'username':username,'passwd':password, 'name':'login'}
cookie = cookielib.LWPCookieJar() cookie = http.cookiejar.LWPCookieJar()
params = urllib.urlencode(auth_params) params = urllib.parse.urlencode(auth_params).encode('ascii')
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener) urllib.request.install_opener(opener)
request = urllib2.Request(BASE_URI + 'login', params) request = urllib.request.Request(BASE_URI + 'login', params)
_ = urllib2.urlopen(request) _ = urllib.request.urlopen(request)
return cookie return cookie
def SubmitBuild(branch, extension = "", builder_name = None): def SubmitBuild(branch, extension = "", builder_name = None):
raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'} raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'}
params = urllib.urlencode(raw_params) params = urllib.parse.urlencode(raw_params).encode('ascii')
if not args.docker: if not args.docker:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener) urllib.request.install_opener(opener)
if builder_name == None: if builder_name == None:
builder_name = username + extension builder_name = username + extension
request = urllib2.Request(BUILDERS_URI + builder_name + '/force', params) request = urllib.request.Request(BUILDERS_URI + builder_name + '/force', params)
page = urllib2.urlopen(request) page = urllib.request.urlopen(request)
result = page.read() result = page.read()
if args.verbose: if args.verbose:
print "=== response ===" print("=== response ===")
print result print(result)
print "=== end of response ===" print("=== end of response ===")
if args.docker: if args.docker:
if "<h2>Pending Build Requests:</h2>" in result: if "<h2>Pending Build Requests:</h2>" in result:
print "Build '" + builder_name + "' submitted" print("Build '" + builder_name + "' submitted")
return 0 return 0
else: else:
return -1 return -1
if "Current Builds" in result: if b'Current Builds' in result:
print "Build '" + builder_name + "' submitted" print("Build '" + builder_name + "' submitted")
return 0 return 0
else: else:
return -1 return -1
@ -200,10 +200,10 @@ def SubmitBuild(branch, extension = "", builder_name = None):
# TODO honor the branch argument # TODO honor the branch argument
def FindBuild(branch, extension = "", builder_name = None): def FindBuild(branch, extension = "", builder_name = None):
if builder_name == None: if builder_name == None:
request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/') request = urllib.request.Request(JSON_BUILDERS_URI + username + extension + '/')
else: else:
request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/') request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/')
page = urllib2.urlopen(request) page = urllib.request.urlopen(request)
json_result = json.loads(page.read()) json_result = json.loads(page.read())
# Pending build is unnumbered # Pending build is unnumbered
if json_result["pendingBuilds"]: if json_result["pendingBuilds"]:
@ -218,13 +218,13 @@ def GetBuildStatus(builder, buildid, extension="", builder_name = None):
if builder_name == None: if builder_name == None:
builder_name = username + extension builder_name = username + extension
# https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11 # https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11
request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid)) request = urllib.request.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid))
page = urllib2.urlopen(request) page = urllib.request.urlopen(request)
result = page.read() result = page.read()
if args.verbose: if args.verbose:
print "=== response ===" print("=== response ===")
print result print(result)
print "=== end of response ===" print("=== end of response ===")
json_result = json.loads(result) json_result = json.loads(result)
if json_result["currentStep"]: if json_result["currentStep"]:
return 1 return 1
@ -244,9 +244,9 @@ def WaitForBuildResult(builder, buildid, extension="", builder_name = None):
# return the result # return the result
if res == 0: if res == 0:
print "Build successful for " + builder_name print("Build successful for " + builder_name)
else: else:
print "Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid) print("Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid))
return res return res
# check that github branch and OISF master branch are sync # check that github branch and OISF master branch are sync
@ -254,23 +254,23 @@ if not args.local:
ret = TestGithubSync(args.branch) ret = TestGithubSync(args.branch)
if ret != 0: if ret != 0:
if ret == -2: if ret == -2:
print "Branch " + args.branch + " is not pushed to Github." print("Branch " + args.branch + " is not pushed to Github.")
sys.exit(-1) sys.exit(-1)
if args.norebase: if args.norebase:
print "Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option." print("Branch " + args.branch + " is not in sync with corresponding Github branch. Continuing due to --norebase option.")
else: else:
print "Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed." print("Branch " + args.branch + " is not in sync with corresponding Github branch. Push may be needed.")
sys.exit(-1) sys.exit(-1)
if TestRepoSync(args.branch) != 0: if TestRepoSync(args.branch) != 0:
if args.norebase: if args.norebase:
print "Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option." print("Branch " + args.branch + " is not in sync with OISF's master branch. Continuing due to --norebase option.")
else: else:
print "Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed." print("Branch " + args.branch + " is not in sync with OISF's master branch. Rebase needed.")
sys.exit(-1) sys.exit(-1)
def CreateContainer(): def CreateContainer():
# FIXME check if existing # FIXME check if existing
print "Pulling docking image, first run should take long" print("Pulling docking image, first run should take long")
if GOT_DOCKERPY_API < 2: if GOT_DOCKERPY_API < 2:
cli = Client() cli = Client()
cli.pull('regit/suri-buildbot') cli.pull('regit/suri-buildbot')
@ -279,13 +279,13 @@ def CreateContainer():
cli = DockerClient() cli = DockerClient()
cli.images.pull('regit/suri-buildbot') cli.images.pull('regit/suri-buildbot')
suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
print "Using base src dir: " + suri_src_dir print("Using base src dir: " + suri_src_dir)
cli.containers.create('regit/suri-buildbot', name='suri-buildbot', ports={'8010/tcp': 8010, '22/tcp': None} , volumes={suri_src_dir: { 'bind': '/data/oisf', 'mode': 'ro'}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'mode': 'ro'}}, detach = True) cli.containers.create('regit/suri-buildbot', name='suri-buildbot', ports={'8010/tcp': 8010, '22/tcp': None} , volumes={suri_src_dir: { 'bind': '/data/oisf', 'mode': 'ro'}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'mode': 'ro'}}, detach = True)
sys.exit(0) sys.exit(0)
def StartContainer(): def StartContainer():
suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] suri_src_dir = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
print "Using base src dir: " + suri_src_dir print("Using base src dir: " + suri_src_dir)
if GOT_DOCKERPY_API < 2: if GOT_DOCKERPY_API < 2:
cli = Client() cli = Client()
cli.start('suri-buildbot', port_bindings={8010:8010, 22:None}, binds={suri_src_dir: { 'bind': '/data/oisf', 'ro': True}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'ro': True}} ) cli.start('suri-buildbot', port_bindings={8010:8010, 22:None}, binds={suri_src_dir: { 'bind': '/data/oisf', 'ro': True}, os.path.join(suri_src_dir,'qa','docker','buildbot.cfg'): { 'bind': '/data/buildbot/master/master.cfg', 'ro': True}} )
@ -309,11 +309,11 @@ def RmContainer():
try: try:
cli.remove_container('suri-buildbot') cli.remove_container('suri-buildbot')
except: except:
print "Unable to remove suri-buildbot container" print("Unable to remove suri-buildbot container")
try: try:
cli.remove_image('regit/suri-buildbot:latest') cli.remove_image('regit/suri-buildbot:latest')
except: except:
print "Unable to remove suri-buildbot images" print("Unable to remove suri-buildbot images")
else: else:
cli = DockerClient() cli = DockerClient()
cli.containers.get('suri-buildbot').remove() cli.containers.get('suri-buildbot').remove()
@ -331,7 +331,7 @@ if GOT_DOCKER:
RmContainer() RmContainer()
if not args.branch: if not args.branch:
print "You need to specify a branch for this mode" print("You need to specify a branch for this mode")
sys.exit(-1) sys.exit(-1)
# submit buildbot form to build current branch on the devel builder # submit buildbot form to build current branch on the devel builder
@ -339,12 +339,12 @@ if not args.check:
if not args.docker: if not args.docker:
cookie = OpenBuildbotSession() cookie = OpenBuildbotSession()
if cookie == None: if cookie == None:
print "Unable to connect to buildbot with provided credentials" print("Unable to connect to buildbot with provided credentials")
sys.exit(-1) sys.exit(-1)
for build in BUILDERS_LIST: for build in BUILDERS_LIST:
res = SubmitBuild(args.branch, builder_name = build) res = SubmitBuild(args.branch, builder_name = build)
if res == -1: if res == -1:
print "Unable to start build. Check command line parameters" print("Unable to start build. Check command line parameters")
sys.exit(-1) sys.exit(-1)
buildids = {} buildids = {}
@ -356,20 +356,20 @@ if args.docker:
for build in BUILDERS_LIST: for build in BUILDERS_LIST:
buildid = FindBuild(args.branch, builder_name = build) buildid = FindBuild(args.branch, builder_name = build)
if buildid == -1: if buildid == -1:
print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build print("Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build)
elif buildid == -2: elif buildid == -2:
print "No build found for " + BUILDERS_URI + build print("No build found for " + BUILDERS_URI + build)
sys.exit(0) sys.exit(0)
else: else:
if not args.docker: if not args.docker:
print "You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid) print("You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid))
buildids[build] = buildid buildids[build] = buildid
if args.docker: if args.docker:
print "You can watch build progress at " + BASE_URI + "waterfall" print("You can watch build progress at " + BASE_URI + "waterfall")
if len(buildids): if len(buildids):
print "Waiting for build completion" print("Waiting for build completion")
else: else:
sys.exit(0) sys.exit(0)
@ -383,24 +383,24 @@ if args.docker:
buildres = -1 buildres = -1
up_buildids.pop(build, None) up_buildids.pop(build, None)
if len(up_buildids): if len(up_buildids):
remains = " (remaining builds: " + ', '.join(up_buildids.keys()) + ")" remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")"
else: else:
remains = "" remains = ""
print "Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains print("Build failure for " + build + ": " + BUILDERS_URI + build + '/builds/' + str(buildids[build]) + remains)
elif ret == 0: elif ret == 0:
up_buildids.pop(build, None) up_buildids.pop(build, None)
if len(up_buildids): if len(up_buildids):
remains = " (remaining builds: " + ', '.join(up_buildids.keys()) + ")" remains = " (remaining builds: " + ', '.join(list(up_buildids.keys())) + ")"
else: else:
remains = "" remains = ""
print "Build successful for " + build + remains print("Build successful for " + build + remains)
time.sleep(5) time.sleep(5)
buildids = up_buildids buildids = up_buildids
if res == -1: if res == -1:
SendNotification("PRscript failure", "Some builds have failed. Check <a href='" + BASE_URI + "waterfall'>waterfall</a> for results.") SendNotification("PRscript failure", "Some builds have failed. Check <a href='" + BASE_URI + "waterfall'>waterfall</a> for results.")
sys.exit(-1) sys.exit(-1)
else: else:
print "PRscript completed successfully" print("PRscript completed successfully")
SendNotification("PRscript success", "Congrats! All builds have passed.") SendNotification("PRscript success", "Congrats! All builds have passed.")
sys.exit(0) sys.exit(0)
else: else:
@ -411,9 +411,9 @@ else:
if buildres == 0: if buildres == 0:
if not args.norebase and not args.docker: if not args.norebase and not args.docker:
print "You can copy/paste following lines into github PR" print("You can copy/paste following lines into github PR")
for build in buildids: for build in buildids:
print "- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build]) print("- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build]))
SendNotification("OISF PRscript success", "Congrats! All builds have passed.") SendNotification("OISF PRscript success", "Congrats! All builds have passed.")
sys.exit(0) sys.exit(0)
else: else:

Write Preview
Loading…
Cancel
Save