aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

prscript: refactor for docker

Browse Source 
With the current work in progress on docker we need to update the
script to handle this case.

This patch adds two options:
 - -d to run in docker mode. In that case the build is started in
 the local dockerized buildbot instance
 - -l to not test the tested branch synchronization with github.

The -l option allows user to run a complete test without publishing
the code on github and without Internet access.
pull/1405/head
Eric Leblond 12 years ago committed by Victor Julien
parent 3b827fd649
commit dd6f9a6298
1 changed files with 80 additions and 58 deletions
Show Stats Download Patch File Download Diff File

138
qa/prscript.py
Unescape Escape View File

@ -1,5 +1,5 @@
#!/usr/bin/env python
# Copyright(C) 2013 Open Information Security Foundation
# Copyright(C) 2013, 2014, 2015 Open Information Security Foundation
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -27,9 +27,6 @@ import sys
# - buildbot user and password
BASE_URI="https://buildbot.openinfosecfoundation.org/"
BUILDERS_URI=BASE_URI+"builders/"
JSON_BUILDERS_URI=BASE_URI+"json/builders/"
GITHUB_BASE_URI = "https://api.github.com/repos/"
GITHUB_MASTER_URI = "https://api.github.com/repos/inliniac/suricata/commits?sha=master"
@ -40,12 +37,24 @@ parser.add_argument('-c', '--check', action='store_const', const=True, help='onl
parser.add_argument('-v', '--verbose', action='store_const', const=True, help='verbose output', default=False)
parser.add_argument('--norebase', action='store_const', const=True, help='do not test if branch is in sync with master', default=False)
parser.add_argument('-r', '--repository', dest='repository', default='suricata', help='name of suricata repository on github')
parser.add_argument('-d', '--docker', action='store_const', const=True, help='use docker based testing', default=False)
parser.add_argument('-l', '--local', action='store_const', const=True, help='local testing before github push', default=False)
parser.add_argument('branch', metavar='branch', help='github branch to build')
args = parser.parse_args()
username = args.username
password = args.password
cookie = None
if args.docker:
BASE_URI="http://localhost:8010/"
BUILDERS_LIST = ["build", "clang", "debug", "features", "profiling", "pcaps"]
else:
BUILDERS_LIST = [username, username + "-pcap"]
BUILDERS_URI=BASE_URI+"builders/"
JSON_BUILDERS_URI=BASE_URI+"json/builders/"
def TestRepoSync(branch):
request = urllib2.Request(GITHUB_MASTER_URI)
page = urllib2.urlopen(request)
@ -72,12 +81,15 @@ def OpenBuildbotSession():
return cookie
def SubmitBuild(branch, extension = ""):
def SubmitBuild(branch, extension = "", builder_name = None):
raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'}
params = urllib.urlencode(raw_params)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
request = urllib2.Request(BUILDERS_URI + username + extension + '/force', params)
if not args.docker:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
urllib2.install_opener(opener)
if builder_name == None:
builder_name = username + extension
request = urllib2.Request(BUILDERS_URI + builder_name + '/force', params)
page = urllib2.urlopen(request)
result = page.read()
@ -85,15 +97,24 @@ def SubmitBuild(branch, extension = ""):
print "=== response ==="
print result
print "=== end of response ==="
if args.docker:
if "<h2>Pending Build Requests:</h2>" in result:
print "Build '" + builder_name + "' submitted"
return 0
else:
return -1
if "Current Builds" in result:
print "Build submitted"
print "Build '" + builder_name + "' submitted"
return 0
else:
return -1
# TODO honor the branch argument
def FindBuild(branch, extension = ""):
request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/')
def FindBuild(branch, extension = "", builder_name = None):
if builder_name == None:
request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/')
else:
request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/')
page = urllib2.urlopen(request)
json_result = json.loads(page.read())
# Pending build is unnumbered
@ -105,9 +126,12 @@ def FindBuild(branch, extension = ""):
return json_result["cachedBuilds"][-1]
return -2
def GetBuildStatus(builder, buildid, extension=""):
# https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11
request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/builds/' + str(buildid))
def GetBuildStatus(builder, buildid, extension="", builder_name = None):
if builder_name == None:
# https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11
request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/builds/' + str(buildid))
else:
request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid))
page = urllib2.urlopen(request)
result = page.read()
if args.verbose:
@ -121,23 +145,25 @@ def GetBuildStatus(builder, buildid, extension=""):
return 0
return -1
def WaitForBuildResult(builder, buildid, extension=""):
def WaitForBuildResult(builder, buildid, extension="", builder_name = None):
# fetch result every 10 secs till task is over
if builder_name == None:
builder_name = username + extension
res = 1
while res == 1:
res = GetBuildStatus(username,buildid, extension=extension)
res = GetBuildStatus(username,buildid, builder_name = builder_name)
if res == 1:
time.sleep(10)
# return the result
if res == 0:
print "Build successful"
print "Build successful for " + builder_name
else:
print "Build failure: " + BUILDERS_URI + username + extension + '/builds/' + str(buildid)
print "Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid)
return res
# check that github branch and inliniac master branch are sync
if TestRepoSync(args.branch) == -1:
if not args.local and TestRepoSync(args.branch) == -1:
if args.norebase:
print "Branch " + args.branch + " is not in sync with inliniac's master branch. Continuing due to --norebase option."
else:
@ -146,54 +172,50 @@ if TestRepoSync(args.branch) == -1:
# submit buildbot form to build current branch on the devel builder
if not args.check:
cookie = OpenBuildbotSession()
if cookie == None:
print "Unable to connect to buildbot with provided credentials"
sys.exit(-1)
res = SubmitBuild(args.branch)
if res == -1:
print "Unable to start build. Check command line parameters"
sys.exit(-1)
res = SubmitBuild(args.branch, extension="-pcap")
if res == -1:
print "Unable to start pcap build. Check command line parameters"
sys.exit(-1)
# get build number and exit if we don't have
buildid = FindBuild(args.branch)
if buildid == -1:
print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + username
elif buildid == -2:
print "No build found for " + BUILDERS_URI + username
sys.exit(0)
else:
print "You can watch build progress at " + BUILDERS_URI + username + "/builds/" + str(buildid)
if not args.docker:
cookie = OpenBuildbotSession()
if cookie == None:
print "Unable to connect to buildbot with provided credentials"
sys.exit(-1)
for build in BUILDERS_LIST:
res = SubmitBuild(args.branch, builder_name = build)
if res == -1:
print "Unable to start build. Check command line parameters"
sys.exit(-1)
buildids = {}
if args.docker:
from time import sleep
print "Waiting 2 seconds"
sleep(2)
# get build number and exit if we don't have
buildidpcap = FindBuild(args.branch, extension = "-pcap")
if buildidpcap == -1:
print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + username + "-pcap"
elif buildidpcap == -2:
print "No build found for " + BUILDERS_URI + username + "-pcap"
sys.exit(0)
else:
print "You can watch build progress at " + BUILDERS_URI + username + "-pcap/builds/" + str(buildidpcap)
for build in BUILDERS_LIST:
buildid = FindBuild(args.branch, builder_name = build)
if buildid == -1:
print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build
elif buildid == -2:
print "No build found for " + BUILDERS_URI + build
sys.exit(0)
else:
print "You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid)
buildids[build] = buildid
if buildid != -1 or buildidpcap != -1:
if len(buildids):
print "Waiting for build completion"
else:
sys.exit(0)
res = 0
if buildid != -1:
res = WaitForBuildResult(username, buildid)
if buildidpcap != -1:
res += WaitForBuildResult(username, buildidpcap, extension="-pcap")
for build in buildids:
res = WaitForBuildResult(build, buildids[build], builder_name = build)
if res == 0:
if not args.norebase:
if not args.norebase and not args.docker:
print "You can copy/paste following lines into github PR"
print "- PR build: " + BUILDERS_URI + username + "/builds/" + str(buildid)
print "- PR pcaps: " + BUILDERS_URI + username + "-pcap/builds/" + str(buildidpcap)
for build in buildids:
print "- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build])
sys.exit(0)
else:
sys.exit(-1)

Write Preview
Loading…
Cancel
Save