From 617a0442519d3c5602a4228a620fdd9e49ea33a3 Mon Sep 17 00:00:00 2001 From: Benny Prijono Date: Mon, 29 Dec 2008 14:56:32 +0000 Subject: Added cdash framework for pjsip tests. Currently there is only test framework for GNU git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@2399 74dad513-b988-da41-8d7b-12977e46ad98 --- tests/cdash/README.TXT | 50 ++++++++ tests/cdash/builder.py | 262 +++++++++++++++++++++++++++++++++++++++++ tests/cdash/cfg_gnu.py | 34 ++++++ tests/cdash/cfg_site_sample.py | 25 ++++ tests/cdash/inc_test.py | 1 + tests/cdash/main.py | 20 ++++ 6 files changed, 392 insertions(+) create mode 100644 tests/cdash/README.TXT create mode 100644 tests/cdash/builder.py create mode 100644 tests/cdash/cfg_gnu.py create mode 100644 tests/cdash/cfg_site_sample.py create mode 100644 tests/cdash/inc_test.py create mode 100644 tests/cdash/main.py (limited to 'tests/cdash') diff --git a/tests/cdash/README.TXT b/tests/cdash/README.TXT new file mode 100644 index 00000000..0d09965a --- /dev/null +++ b/tests/cdash/README.TXT @@ -0,0 +1,50 @@ + + PJSIP CDASH AUTOMATED TESTS + -------------------------------- + + +1. What is this + + This directory contains the scripts to run the automated, Python based tests +of PJSIP source codes, across platforms, and submit the test results to a CDash +test monitoring dashboard. + + Stuffs that are included in the test scope: + - configure (for GNU platforms, e.g. Linux, msys, and MacOS X) + - build + - standard unit tests (pjlib-test, pjsip-test, etc.) + - pjsua's Python based blackbox tests + + +2. Requirements + + To run this test, you will need: + - Python (2.4 or later) + - curl (http://curl.haxx.se) + - a CDash server to receive test submissions (http://www.cdash.org) + - ccdash to submit the tests (http://trac.pjsip.org/ccdash) + + +3. Configuration + + Create a Python configuration file by copying from "cfg_site_sample.py". +Save it as "cfg_site.py". You may create more than one configurations for your +site and save them as different files. + + +4. Running + + To execute tests for GNU based targets: + + $ python main.py cfg_gnu + + + To execute tests for MSVC based target: + + $ python main.py cfg_msvc + + + If you have a different site configuration file, you may specify it in the +arguments, e.g.: + + $ python main.py cfg_gnu my_site_config diff --git a/tests/cdash/builder.py b/tests/cdash/builder.py new file mode 100644 index 00000000..6336a3db --- /dev/null +++ b/tests/cdash/builder.py @@ -0,0 +1,262 @@ +import ccdash +import os +import platform +import re +import subprocess +import sys +import time + +class Operation: + """\ + The Operation class describes the individual ccdash operation to be + performed. + + """ + # Types: + UPDATE = "update" # Update operation + CONFIGURE = "configure" # Configure operation + BUILD = "build" # Build operation + TEST = "test" # Unit test operation + + def __init__(self, type, cmdline, name="", wdir=""): + self.type = type + self.cmdline = cmdline + self.name = name + self.wdir = wdir + if self.type==self.TEST and not self.name: + raise "name required for tests" + + def encode(self, base_dir): + s = [self.type] + if self.type == self.TEST: + s.append(self.name) + if self.type != self.UPDATE: + s.append(self.cmdline) + s.append("-w") + if self.wdir: + s.append(base_dir + "/" + self.wdir) + else: + s.append(base_dir) + return s + + +# +# Update operation +# +update_ops = [Operation(Operation.UPDATE, "")] + +# +# The standard library tests (e.g. pjlib-test, pjsip-test, etc.) +# +std_test_ops= [ + Operation(Operation.TEST, "./pjlib-test-$SUFFIX", name="pjlib test", + wdir="pjlib/bin"), + Operation(Operation.TEST, "./pjlib-util-test-$SUFFIX", + name="pjlib-util test", wdir="pjlib-util/bin"), + Operation(Operation.TEST, "./pjnath-test-$SUFFIX", name="pjnath test", + wdir="pjnath/bin"), + Operation(Operation.TEST, "./pjmedia-test-$SUFFIX", name="pjmedia test", + wdir="pjmedia/bin"), + Operation(Operation.TEST, "./pjsip-test-$SUFFIX", name="pjsip test", + wdir="pjsip/bin") +] + +# +# These are operations to build the software on GNU/Posix systems +# +gnu_build_ops = [ + Operation(Operation.CONFIGURE, "./configure"), + Operation(Operation.BUILD, "make distclean; make dep && make; cd pjsip-apps/src/python; python setup.py clean build"), + #Operation(Operation.BUILD, "python setup.py clean build", + # wdir="pjsip-apps/src/python") +] + +# +# These are pjsua Python based unit test operations +# +def build_pjsua_test_ops(): + ops = [] + cwd = os.getcwd() + os.chdir("../pjsua") + os.system("python runall.py --list > list") + f = open("list", "r") + for e in f: + e = e.rstrip("\r\n ") + (mod,param) = e.split(None,2) + name = mod[4:mod.find(".py")] + "_" + \ + param[param.find("/")+1:param.find(".py")] + ops.append(Operation(Operation.TEST, "python run.py " + e, name=name, + wdir="tests/pjsua")) + os.chdir(cwd) + return ops + +# +# Get gcc version +# +def gcc_version(gcc): + proc = subprocess.Popen(gcc + " -v", stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True) + ver = "" + while True: + s = proc.stdout.readline() + if not s: + break + if s.find("gcc version") >= 0: + ver = s.split(None, 3)[2] + break + proc.wait() + return "gcc-" + ver + +# +# Test config +# +class BaseConfig: + def __init__(self, base_dir, url, site, group, options=None): + self.base_dir = base_dir + self.url = url + self.site = site + self.group = group + self.options = options + +# +# Base class for test configurator +# +class TestBuilder: + def __init__(self, config, build_config_name="", + user_mak="", config_site="", exclude=[], not_exclude=[]): + self.config = config # BaseConfig instance + self.build_config_name = build_config_name # Optional build suffix + self.user_mak = user_mak # To be put in user.mak + self.config_site = config_site # To be put in config_s.. + self.saved_user_mak = "" # To restore user.mak + self.saved_config_site = "" # To restore config_s.. + self.exclude = exclude # List of exclude pattern + self.not_exclude = not_exclude # List of include pattern + self.ccdash_args = [] # ccdash cmd line + + def stamp(self): + return time.strftime("%Y%m%d-%H%M", time.localtime()) + + def pre_action(self): + # Override user.mak + name = self.config.base_dir + "/user.mak" + if os.access(name, os.F_OK): + f = open(name, "r") + self.saved_user_mak = f.read() + f.close() + if True: + f = open(name, "wt") + f.write(self.user_mak) + f.close() + # Override config_site.h + name = self.config.base_dir + "/pjlib/include/pj/config_site.h" + if os.access(name, os.F_OK): + f = open(name, "r") + self.saved_config_site= f.read() + f.close() + if True: + f = open(name, "wt") + f.write(self.config_site) + f.close() + + + def post_action(self): + # Restore user.mak + name = self.config.base_dir + "/user.mak" + if self.saved_user_mak: + f = open(name, "wt") + f.write(self.saved_user_mak) + f.close() + else: + os.remove(name) + # Restore config_site.h + name = self.config.base_dir + "/pjlib/include/pj/config_site.h" + if self.saved_config_site: + f = open(name, "wt") + f.write(self.saved_config_site) + f.close() + else: + os.remove(name) + + def build_tests(self): + # This should be overridden by subclasses + pass + + def execute(self): + if len(self.ccdash_args)==0: + self.build_tests() + self.pre_action() + counter = 0 + for a in self.ccdash_args: + # Check if this test is in exclusion list + fullcmd = " ".join(a) + excluded = False + included = False + for pat in self.exclude: + if re.search(pat, fullcmd) != None: + excluded = True + break + if excluded: + for pat in self.not_exclude: + if re.search(pat, fullcmd) != None: + included = True + break + if excluded and not included: + print "Skipping test '%s'.." % (fullcmd) + continue + + #a.extend(["-o", "/tmp/xx" + a[0] + ".xml"]) + #print a + #a = ["ccdash.py"].extend(a) + b = ["ccdash.py"] + b.extend(a) + a = b + #print a + ccdash.main(a) + counter = counter + 1 + self.post_action() + + +# +# GNU test configurator +# +class GNUTestBuilder(TestBuilder): + def __init__(self, config, build_config_name="", user_mak="", \ + config_site="", cross_compile="", exclude=[], not_exclude=[]): + TestBuilder.__init__(self, config, build_config_name=build_config_name, + user_mak=user_mak, config_site=config_site, + exclude=exclude, not_exclude=not_exclude) + self.cross_compile = cross_compile + if self.cross_compile and self.cross_compile[-1] != '-': + self.cross_compile.append("-") + + def build_tests(self): + if self.cross_compile: + suffix = self.cross_compile + build_name = suffix + gcc_version(self.cross_compile + "gcc") + else: + proc = subprocess.Popen(self.config.base_dir+"/config.guess", + stdout=subprocess.PIPE) + suffix = proc.stdout.readline().rstrip(" \r\n") + build_name = suffix+"-"+gcc_version(self.cross_compile + "gcc") + + if self.build_config_name: + build_name = build_name + "-" + self.build_config_name + cmds = [] + cmds.extend(update_ops) + cmds.extend(gnu_build_ops) + cmds.extend(std_test_ops) + cmds.extend(build_pjsua_test_ops()) + self.ccdash_args = [] + for c in cmds: + c.cmdline = c.cmdline.replace("$SUFFIX", suffix) + args = c.encode(self.config.base_dir) + args.extend(["-U", self.config.url, + "-S", self.config.site, + "-T", self.stamp(), + "-B", build_name, + "-G", self.config.group]) + args.extend(self.config.options) + self.ccdash_args.append(args) + + diff --git a/tests/cdash/cfg_gnu.py b/tests/cdash/cfg_gnu.py new file mode 100644 index 00000000..55bb82bc --- /dev/null +++ b/tests/cdash/cfg_gnu.py @@ -0,0 +1,34 @@ +import builder +import os +import sys + +# Each configurator must export this function +def create_builder(args): + # (optional) args format: + # site configuration module. If not specified, "cfg_site" is implied + + if len(args)>0: + file = args[0] + else: + file = "cfg_site" + + if os.access(file+".py", os.F_OK) == False: + print "Error: file '%s.py' doesn't exist." % (file) + sys.exit(1) + + cfg_site = __import__(file) + test_cfg = builder.BaseConfig(cfg_site.BASE_DIR, \ + cfg_site.URL, \ + cfg_site.SITE_NAME, \ + cfg_site.GROUP, \ + cfg_site.OPTIONS) + + builders = [ + builder.GNUTestBuilder(test_cfg, build_config_name="default", + user_mak="export CFLAGS+=-Wall\n", + config_site="#define PJ_TODO(x)\n", + exclude=cfg_site.EXCLUDE, + not_exclude=cfg_site.NOT_EXCLUDE) + ] + + return builders diff --git a/tests/cdash/cfg_site_sample.py b/tests/cdash/cfg_site_sample.py new file mode 100644 index 00000000..147aa496 --- /dev/null +++ b/tests/cdash/cfg_site_sample.py @@ -0,0 +1,25 @@ +import builder + +# Your site name +SITE_NAME="Newham2" + +# The URL where tests will be submitted to +URL = "http://192.168.0.2/dash/submit.php?project=PJSIP" + +# Test group +GROUP = "Experimental" + +# PJSIP base directory +BASE_DIR = "/root/project/pjproject" + +# List of additional ccdash options +#OPTIONS = ["-o", "out.xml", "-y"] +OPTIONS = ["-o", "out.xml"] + +# List of regular expression of test patterns to be excluded +EXCLUDE = [".*"] + +# List of regular expression of test patterns to be included (even +# if they match EXCLUDE patterns) +NOT_EXCLUDE = ["run.py mod_run.*100_simple"] +#"configure", "update", "build.*make", "build", "run.py mod_run.*100_simple"] diff --git a/tests/cdash/inc_test.py b/tests/cdash/inc_test.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/cdash/inc_test.py @@ -0,0 +1 @@ + diff --git a/tests/cdash/main.py b/tests/cdash/main.py new file mode 100644 index 00000000..9f707bd7 --- /dev/null +++ b/tests/cdash/main.py @@ -0,0 +1,20 @@ +import sys + +if len(sys.argv)==1: + print "Usage: main.py cfg_file [cfg_site]" + print "Example:" + print " main.py cfg_gnu" + print " main.py cfg_gnu custom_cfg_site" + sys.exit(1) + + +args = [] +args.extend(sys.argv) +args.remove(args[1]) +args.remove(args[0]) + +cfg_file = __import__(sys.argv[1]) +builders = cfg_file.create_builder(args) + +for builder in builders: + builder.execute() -- cgit v1.2.3