suricata/qa/docker/buildbot.cfg

# -*- python -*-
# ex: set syntax=python:

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### BUILDSLAVES

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
c['slaves'] = [BuildSlave("buildslave", "Suridocker")]

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = 9989

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

from buildbot.changes.gitpoller import GitPoller
c['change_source'] = []
c['change_source'].append(GitPoller(
        '/data/oisf/.git/',
        workdir='gitpoller-workdir', branches = ['master'],
        pollinterval=300, project='suricata'))

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'runtests' build

from buildbot.schedulers.basic import SingleBranchScheduler
#from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.changes import filter
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
                            name="master",
                            change_filter=filter.ChangeFilter(branch='master'),
                            treeStableTimer=None,
                            builderNames=["features","profiling","clang"]))

#c['schedulers'].append(ForceScheduler(
#                            name="force",
#                            builderNames=["builds","debug"]))

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

from buildbot.process.factory import BuildFactory
#from buildbot.steps.source.git import Git
from buildbot.steps.source import Git
from buildbot.steps.shell import ShellCommand

def SuriBuildFactory(repo='/data/oisf/'):
    factory = BuildFactory()
    factory.addStep(Git(repourl=repo, mode='copy'))
    factory.addStep(ShellCommand(command=["rm", "-rf", "libhtp"]))
    factory.addStep(ShellCommand(command=["git", "clone", "-b", "0.5.x", "/data/oisf/libhtp/.git/", "libhtp"]))
    return factory

factory = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory.addStep(ShellCommand(command=["./autogen.sh"]))
factory.addStep(ShellCommand(command=["./configure"]))
factory.addStep(ShellCommand(command=["make"]))
factory.addStep(ShellCommand(command=["make", "clean"]))
#factory.addStep(ShellCommand(command=["make", "distcheck"]))

factory_devel = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory_devel.addStep(ShellCommand(command=["./autogen.sh"]))
factory_devel.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests"]))
factory_devel.addStep(ShellCommand(command=["make"]))
factory_devel.addStep(ShellCommand(command=["make", "clean"]))
#factory_devel.addStep(ShellCommand(command=["make", "distcheck"], env={'DISTCHECK_CONFIGURE_FLAGS': "--enable-debug --enable-unittests"}))

factory_profiling = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory_profiling.addStep(ShellCommand(command=["./autogen.sh"]))
factory_profiling.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-profiling","--enable-unittests"]))
factory_profiling.addStep(ShellCommand(command=["make"]))
factory_profiling.addStep(ShellCommand(command=["make", "clean"]))
#factory_profiling.addStep(ShellCommand(command=["make", "distcheck"],env={'DISTCHECK_CONFIGURE_FLAGS': "--enable-debug --enable-profiling --enable-unittests"}))

factory_clang = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory_clang.addStep(ShellCommand(command=["./autogen.sh"]))
#factory_clang.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","CC=clang","CFLAGS=-fsanitize=address"]))
factory_clang.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","CC=clang","ac_cv_func_malloc_0_nonnull=yes","ac_cv_func_realloc_0_nonnull=yes"]))
factory_clang.addStep(ShellCommand(command=["make"]))
factory_clang.addStep(ShellCommand(command=["make", "clean"]))

factory_clang_32 = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory_clang_32.addStep(ShellCommand(command=["./autogen.sh"]))
factory_clang_32.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","CC=clang","CFLAGS=-fsanitize=address","ac_cv_func_malloc_0_nonnull=yes","ac_cv_func_realloc_0_nonnull=yes"]))
factory_clang_32.addStep(ShellCommand(command=["make"]))
factory_clang_32.addStep(ShellCommand(command=["make", "clean"]))

factory_features = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory_features.addStep(ShellCommand(command=["./autogen.sh"]))
factory_features.addStep(ShellCommand(command=["./configure","--enable-debug","--enable-unittests","--enable-nfqueue","--enable-nflog", "--enable-lua", "--enable-prelude"]))
factory_features.addStep(ShellCommand(command=["make"]))
factory_features.addStep(ShellCommand(command=["make", "clean"]))
import psutil
factory_features.addStep(ShellCommand(command=["make", "distcheck"],env={'DISTCHECK_CONFIGURE_FLAGS': "--enable-debug --enable-unittests --enable-nfqueue --enable-nflog --enable-lua --enable-prelude", "CONCURRENCY_LEVEL": str(psutil.cpu_count())}))

import os
PCAP_PATH='/data/oisf/qa/docker/pcaps/'
(_, _, pcaps_list) = os.walk(PCAP_PATH).next()
pcaps_list = [ os.path.join(PCAP_PATH, pcap) for pcap in pcaps_list if pcap.endswith(".pcap") ]

factory_stress_pcap = SuriBuildFactory()
# run the tests (note that this will require that 'trial' is installed)
factory_stress_pcap.addStep(ShellCommand(command=["./autogen.sh"]))
factory_stress_pcap.addStep(ShellCommand(command=["./configure","--enable-debug-validation"],env={"CFLAGS" : "-fsanitize=address -fno-omit-frame-pointer"}))
factory_stress_pcap.addStep(ShellCommand(command=["make"]))
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "make","install"]))
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "rm", "-f", "/usr/local/etc/suricata/suricata.yaml"]))
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "make","install-conf"]))
factory_stress_pcap.addStep(ShellCommand(command=["make","clean"]))
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "ldconfig"]))
for pfile in pcaps_list:
    factory_stress_pcap.addStep(ShellCommand(command=["sudo", "/usr/local/bin/suricata","-r",pfile,"--init-errors-fatal","-S","/data/oisf/rules/http-events.rules"]))
factory_stress_pcap.addStep(ShellCommand(command=["sudo", "rm", "-rf", "/usr/local/var/log/suricata/"]))

from buildbot.config import BuilderConfig

def SuriBuilderConfig(*args, **kwargs):
    if not kwargs.has_key('category'):
        kwargs['category']='default'
    return BuilderConfig(*args, **kwargs)

c['builders'] = []

c['builders'].append(
    SuriBuilderConfig(name="gcc",
      slavename="buildslave",
      factory=factory))
c['schedulers'].append(SingleBranchScheduler(
                            name="build",
                            change_filter=filter.ChangeFilter(branch='master'),
                            treeStableTimer=None,
                            builderNames=["gcc"]))

c['builders'].append(
    SuriBuilderConfig(name="debug",
      slavename="buildslave",
      factory=factory_devel))
c['schedulers'].append(SingleBranchScheduler(
                            name="debug",
                            change_filter=filter.ChangeFilter(branch='master'),
                            treeStableTimer=None,
                            builderNames=["debug"]))

c['builders'].append(
    SuriBuilderConfig(name="profiling",
      slavename="buildslave",
      factory=factory_profiling))
c['builders'].append(
    SuriBuilderConfig(name="clang",
      slavename="buildslave",
      factory=factory_clang_32))
c['builders'].append(
    SuriBuilderConfig(name="features",
      slavename="buildslave",
      factory=factory_features))
c['builders'].append(
    SuriBuilderConfig(name="pcaps",
      slavename="buildslave",
      factory=factory_stress_pcap))

from buildbot import locks
build_lock = locks.SlaveLock("slave_builds", maxCount = 1)


from buildbot.schedulers.forcesched import *
c['schedulers'].append(ForceScheduler(name="force", builderNames = [ builder.getConfigDict()['name'] for builder in c['builders'] ]))

c['status'] = []

from buildbot.status import html
from buildbot.status.web import authz, auth

authz_cfg=authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    #auth=auth.BasicAuth(users),
    gracefulShutdown = False,
    forceBuild = True, # use this to test your slave once it is set up
    forceAllBuilds = True,
    pingBuilder = True,
    stopBuild = True,
    stopAllBuilds = True,
    cancelPendingBuild = True,
)
c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "Suricata"
c['titleURL'] = "https://redmine.openinfosecfoundation.org/projects/suricata"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "http://localhost:8010/"

####### DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url' : "sqlite:///state.sqlite",
}