From dd6f9a6298e14e655d264467d181bda90844ce5f Mon Sep 17 00:00:00 2001 From: Eric Leblond Date: Tue, 10 Mar 2015 13:15:15 +0100 Subject: [PATCH] prscript: refactor for docker With the current work in progress on docker we need to update the script to handle this case. This patch adds two options: - -d to run in docker mode. In that case the build is started in the local dockerized buildbot instance - -l to not test the tested branch synchronization with github. The -l option allows user to run a complete test without publishing the code on github and without Internet access. --- qa/prscript.py | 138 ++++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 58 deletions(-) diff --git a/qa/prscript.py b/qa/prscript.py index 142ad6802a..cc29a2a16f 100755 --- a/qa/prscript.py +++ b/qa/prscript.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright(C) 2013 Open Information Security Foundation +# Copyright(C) 2013, 2014, 2015 Open Information Security Foundation # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,9 +27,6 @@ import sys # - buildbot user and password BASE_URI="https://buildbot.openinfosecfoundation.org/" -BUILDERS_URI=BASE_URI+"builders/" -JSON_BUILDERS_URI=BASE_URI+"json/builders/" - GITHUB_BASE_URI = "https://api.github.com/repos/" GITHUB_MASTER_URI = "https://api.github.com/repos/inliniac/suricata/commits?sha=master" @@ -40,12 +37,24 @@ parser.add_argument('-c', '--check', action='store_const', const=True, help='onl parser.add_argument('-v', '--verbose', action='store_const', const=True, help='verbose output', default=False) parser.add_argument('--norebase', action='store_const', const=True, help='do not test if branch is in sync with master', default=False) parser.add_argument('-r', '--repository', dest='repository', default='suricata', help='name of suricata repository on github') +parser.add_argument('-d', '--docker', action='store_const', const=True, help='use docker based testing', default=False) +parser.add_argument('-l', '--local', action='store_const', const=True, help='local testing before github push', default=False) parser.add_argument('branch', metavar='branch', help='github branch to build') args = parser.parse_args() username = args.username password = args.password cookie = None +if args.docker: + BASE_URI="http://localhost:8010/" + BUILDERS_LIST = ["build", "clang", "debug", "features", "profiling", "pcaps"] +else: + BUILDERS_LIST = [username, username + "-pcap"] + + +BUILDERS_URI=BASE_URI+"builders/" +JSON_BUILDERS_URI=BASE_URI+"json/builders/" + def TestRepoSync(branch): request = urllib2.Request(GITHUB_MASTER_URI) page = urllib2.urlopen(request) @@ -72,12 +81,15 @@ def OpenBuildbotSession(): return cookie -def SubmitBuild(branch, extension = ""): +def SubmitBuild(branch, extension = "", builder_name = None): raw_params = {'branch':branch,'reason':'Testing ' + branch, 'name':'force_build', 'forcescheduler':'force'} params = urllib.urlencode(raw_params) - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) - urllib2.install_opener(opener) - request = urllib2.Request(BUILDERS_URI + username + extension + '/force', params) + if not args.docker: + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie)) + urllib2.install_opener(opener) + if builder_name == None: + builder_name = username + extension + request = urllib2.Request(BUILDERS_URI + builder_name + '/force', params) page = urllib2.urlopen(request) result = page.read() @@ -85,15 +97,24 @@ def SubmitBuild(branch, extension = ""): print "=== response ===" print result print "=== end of response ===" + if args.docker: + if "

Pending Build Requests:

" in result: + print "Build '" + builder_name + "' submitted" + return 0 + else: + return -1 if "Current Builds" in result: - print "Build submitted" + print "Build '" + builder_name + "' submitted" return 0 else: return -1 # TODO honor the branch argument -def FindBuild(branch, extension = ""): - request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/') +def FindBuild(branch, extension = "", builder_name = None): + if builder_name == None: + request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/') + else: + request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/') page = urllib2.urlopen(request) json_result = json.loads(page.read()) # Pending build is unnumbered @@ -105,9 +126,12 @@ def FindBuild(branch, extension = ""): return json_result["cachedBuilds"][-1] return -2 -def GetBuildStatus(builder, buildid, extension=""): - # https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11 - request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/builds/' + str(buildid)) +def GetBuildStatus(builder, buildid, extension="", builder_name = None): + if builder_name == None: + # https://buildbot.suricata-ids.org/json/builders/build%20deb6/builds/11 + request = urllib2.Request(JSON_BUILDERS_URI + username + extension + '/builds/' + str(buildid)) + else: + request = urllib2.Request(JSON_BUILDERS_URI + builder_name + '/builds/' + str(buildid)) page = urllib2.urlopen(request) result = page.read() if args.verbose: @@ -121,23 +145,25 @@ def GetBuildStatus(builder, buildid, extension=""): return 0 return -1 -def WaitForBuildResult(builder, buildid, extension=""): +def WaitForBuildResult(builder, buildid, extension="", builder_name = None): # fetch result every 10 secs till task is over + if builder_name == None: + builder_name = username + extension res = 1 while res == 1: - res = GetBuildStatus(username,buildid, extension=extension) + res = GetBuildStatus(username,buildid, builder_name = builder_name) if res == 1: time.sleep(10) # return the result if res == 0: - print "Build successful" + print "Build successful for " + builder_name else: - print "Build failure: " + BUILDERS_URI + username + extension + '/builds/' + str(buildid) + print "Build failure for " + builder_name + ": " + BUILDERS_URI + builder_name + '/builds/' + str(buildid) return res # check that github branch and inliniac master branch are sync -if TestRepoSync(args.branch) == -1: +if not args.local and TestRepoSync(args.branch) == -1: if args.norebase: print "Branch " + args.branch + " is not in sync with inliniac's master branch. Continuing due to --norebase option." else: @@ -146,54 +172,50 @@ if TestRepoSync(args.branch) == -1: # submit buildbot form to build current branch on the devel builder if not args.check: - cookie = OpenBuildbotSession() - if cookie == None: - print "Unable to connect to buildbot with provided credentials" - sys.exit(-1) - res = SubmitBuild(args.branch) - if res == -1: - print "Unable to start build. Check command line parameters" - sys.exit(-1) - res = SubmitBuild(args.branch, extension="-pcap") - if res == -1: - print "Unable to start pcap build. Check command line parameters" - sys.exit(-1) - -# get build number and exit if we don't have -buildid = FindBuild(args.branch) -if buildid == -1: - print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + username -elif buildid == -2: - print "No build found for " + BUILDERS_URI + username - sys.exit(0) -else: - print "You can watch build progress at " + BUILDERS_URI + username + "/builds/" + str(buildid) + if not args.docker: + cookie = OpenBuildbotSession() + if cookie == None: + print "Unable to connect to buildbot with provided credentials" + sys.exit(-1) + for build in BUILDERS_LIST: + res = SubmitBuild(args.branch, builder_name = build) + if res == -1: + print "Unable to start build. Check command line parameters" + sys.exit(-1) + +buildids = {} + +if args.docker: + from time import sleep + print "Waiting 2 seconds" + sleep(2) # get build number and exit if we don't have -buildidpcap = FindBuild(args.branch, extension = "-pcap") -if buildidpcap == -1: - print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + username + "-pcap" -elif buildidpcap == -2: - print "No build found for " + BUILDERS_URI + username + "-pcap" - sys.exit(0) -else: - print "You can watch build progress at " + BUILDERS_URI + username + "-pcap/builds/" + str(buildidpcap) +for build in BUILDERS_LIST: + buildid = FindBuild(args.branch, builder_name = build) + if buildid == -1: + print "Pending build tracking is not supported. Follow build by browsing " + BUILDERS_URI + build + elif buildid == -2: + print "No build found for " + BUILDERS_URI + build + sys.exit(0) + else: + print "You can watch build progress at " + BUILDERS_URI + build + "/builds/" + str(buildid) + buildids[build] = buildid -if buildid != -1 or buildidpcap != -1: +if len(buildids): print "Waiting for build completion" +else: + sys.exit(0) res = 0 -if buildid != -1: - res = WaitForBuildResult(username, buildid) - -if buildidpcap != -1: - res += WaitForBuildResult(username, buildidpcap, extension="-pcap") +for build in buildids: + res = WaitForBuildResult(build, buildids[build], builder_name = build) if res == 0: - if not args.norebase: + if not args.norebase and not args.docker: print "You can copy/paste following lines into github PR" - print "- PR build: " + BUILDERS_URI + username + "/builds/" + str(buildid) - print "- PR pcaps: " + BUILDERS_URI + username + "-pcap/builds/" + str(buildidpcap) + for build in buildids: + print "- PR " + build + ": " + BUILDERS_URI + build + "/builds/" + str(buildids[build]) sys.exit(0) else: sys.exit(-1)