Added cdash framework for pjsip tests. Currently there is only test framework for GNU

git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@2399 74dad513-b988-da41-8d7b-12977e46ad98

6 files changed, 392 insertions, 0 deletions

diff --git a/tests/cdash/README.TXT b/tests/cdash/README.TXT
new file mode 100644
index 00000000..0d09965a
--- /dev/null
+++ b/tests/cdash/README.TXT
@@ -0,0 +1,50 @@
+                                
+                           PJSIP CDASH AUTOMATED TESTS
+                        --------------------------------
+
+
+1. What is this
+
+   This directory contains the scripts to run the automated, Python based tests
+of PJSIP source codes, across platforms, and submit the test results to a CDash
+test monitoring dashboard.
+
+   Stuffs that are included in the test scope:
+    - configure (for GNU platforms, e.g. Linux, msys, and MacOS X)
+    - build
+    - standard unit tests (pjlib-test, pjsip-test, etc.)
+    - pjsua's Python based blackbox tests
+
+
+2. Requirements
+
+   To run this test, you will need:
+    - Python (2.4 or later)
+    - curl (http://curl.haxx.se)
+    - a CDash server to receive test submissions (http://www.cdash.org)
+    - ccdash to submit the tests (http://trac.pjsip.org/ccdash)
+
+
+3. Configuration
+
+   Create a Python configuration file by copying from "cfg_site_sample.py". 
+Save it as "cfg_site.py". You may create more than one configurations for your
+site and save them as different files.
+
+
+4. Running
+
+   To execute tests for GNU based targets:
+
+     $ python main.py cfg_gnu
+
+
+   To execute tests for MSVC based target:
+
+     $ python main.py cfg_msvc
+
+
+   If you have a different site configuration file, you may specify it in the
+arguments, e.g.:
+
+     $ python main.py cfg_gnu my_site_config
diff --git a/tests/cdash/builder.py b/tests/cdash/builder.py
new file mode 100644
index 00000000..6336a3db
--- /dev/null
+++ b/tests/cdash/builder.py
@@ -0,0 +1,262 @@
+import ccdash
+import os
+import platform
+import re
+import subprocess
+import sys
+import time
+
+class Operation:
+    """\
+    The Operation class describes the individual ccdash operation to be 
+    performed.
+
+    """
+    # Types:
+    UPDATE = "update"           # Update operation
+    CONFIGURE = "configure"     # Configure operation
+    BUILD = "build"             # Build operation
+    TEST = "test"               # Unit test operation
+
+    def __init__(self, type, cmdline, name="", wdir=""):
+        self.type = type
+        self.cmdline = cmdline
+        self.name = name
+        self.wdir = wdir
+        if self.type==self.TEST and not self.name:
+            raise "name required for tests"
+
+    def encode(self, base_dir):
+        s = [self.type]
+        if self.type == self.TEST:
+            s.append(self.name)
+        if self.type != self.UPDATE:
+            s.append(self.cmdline)
+        s.append("-w")
+        if self.wdir:
+            s.append(base_dir + "/" + self.wdir)
+        else:
+            s.append(base_dir)
+        return s
+
+
+#
+# Update operation
+#
+update_ops = [Operation(Operation.UPDATE, "")]
+
+#
+# The standard library tests (e.g. pjlib-test, pjsip-test, etc.)
+#
+std_test_ops= [
+    Operation(Operation.TEST, "./pjlib-test-$SUFFIX", name="pjlib test",
+              wdir="pjlib/bin"),
+    Operation(Operation.TEST, "./pjlib-util-test-$SUFFIX", 
+              name="pjlib-util test", wdir="pjlib-util/bin"),
+    Operation(Operation.TEST, "./pjnath-test-$SUFFIX", name="pjnath test",
+              wdir="pjnath/bin"),
+    Operation(Operation.TEST, "./pjmedia-test-$SUFFIX", name="pjmedia test",
+              wdir="pjmedia/bin"),
+    Operation(Operation.TEST, "./pjsip-test-$SUFFIX", name="pjsip test",
+              wdir="pjsip/bin")
+]
+
+#
+# These are operations to build the software on GNU/Posix systems
+#
+gnu_build_ops = [
+    Operation(Operation.CONFIGURE, "./configure"),
+    Operation(Operation.BUILD, "make distclean; make dep && make; cd pjsip-apps/src/python; python setup.py clean build"),
+    #Operation(Operation.BUILD, "python setup.py clean build",
+    #          wdir="pjsip-apps/src/python")
+]
+
+#
+# These are pjsua Python based unit test operations
+#
+def build_pjsua_test_ops():
+    ops = []
+    cwd = os.getcwd()
+    os.chdir("../pjsua")
+    os.system("python runall.py --list > list")
+    f = open("list", "r")
+    for e in f:
+        e = e.rstrip("\r\n ")
+        (mod,param) = e.split(None,2)
+        name = mod[4:mod.find(".py")] + "_" + \
+               param[param.find("/")+1:param.find(".py")]
+        ops.append(Operation(Operation.TEST, "python run.py " + e, name=name,
+                   wdir="tests/pjsua"))
+    os.chdir(cwd)
+    return ops
+
+#
+# Get gcc version
+#
+def gcc_version(gcc):
+    proc = subprocess.Popen(gcc + " -v", stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT, shell=True)
+    ver = ""
+    while True:
+        s = proc.stdout.readline()
+        if not s:
+            break
+        if s.find("gcc version") >= 0:
+            ver = s.split(None, 3)[2]
+            break
+    proc.wait()
+    return "gcc-" + ver
+
+#
+# Test config
+#
+class BaseConfig:
+    def __init__(self, base_dir, url, site, group, options=None):
+        self.base_dir = base_dir
+        self.url = url
+        self.site = site
+        self.group = group
+        self.options = options
+
+#
+# Base class for test configurator
+#
+class TestBuilder:
+    def __init__(self, config, build_config_name="",
+                 user_mak="", config_site="", exclude=[], not_exclude=[]):
+        self.config = config                        # BaseConfig instance
+        self.build_config_name = build_config_name  # Optional build suffix
+        self.user_mak = user_mak                    # To be put in user.mak
+        self.config_site = config_site              # To be put in config_s..
+        self.saved_user_mak = ""                    # To restore user.mak
+        self.saved_config_site = ""                 # To restore config_s..
+        self.exclude = exclude                      # List of exclude pattern
+        self.not_exclude = not_exclude              # List of include pattern
+        self.ccdash_args = []                       # ccdash cmd line
+
+    def stamp(self):
+        return time.strftime("%Y%m%d-%H%M", time.localtime())
+
+    def pre_action(self):
+        # Override user.mak
+        name = self.config.base_dir + "/user.mak"
+        if os.access(name, os.F_OK):
+            f = open(name, "r")
+            self.saved_user_mak = f.read()
+            f.close()
+        if True:
+            f = open(name, "wt")
+            f.write(self.user_mak)
+            f.close()
+        # Override config_site.h
+        name = self.config.base_dir + "/pjlib/include/pj/config_site.h"
+        if os.access(name, os.F_OK):
+            f = open(name, "r")
+            self.saved_config_site= f.read()
+            f.close()
+        if True:
+            f = open(name, "wt")
+            f.write(self.config_site)
+            f.close()
+
+
+    def post_action(self):
+        # Restore user.mak
+        name = self.config.base_dir + "/user.mak"
+        if self.saved_user_mak:
+            f = open(name, "wt")
+            f.write(self.saved_user_mak)
+            f.close()
+        else:
+            os.remove(name)
+        # Restore config_site.h
+        name = self.config.base_dir + "/pjlib/include/pj/config_site.h"
+        if self.saved_config_site:
+            f = open(name, "wt")
+            f.write(self.saved_config_site)
+            f.close()
+        else:
+            os.remove(name)
+
+    def build_tests(self):
+        # This should be overridden by subclasses
+        pass
+
+    def execute(self):
+        if len(self.ccdash_args)==0:
+            self.build_tests()
+        self.pre_action()
+        counter = 0
+        for a in self.ccdash_args:
+            # Check if this test is in exclusion list
+            fullcmd = " ".join(a)
+            excluded = False
+            included = False
+            for pat in self.exclude:
+                if re.search(pat, fullcmd) != None:
+                    excluded = True
+                    break
+            if excluded:
+                for pat in self.not_exclude:
+                    if re.search(pat, fullcmd) != None:
+                        included = True
+                        break
+            if excluded and not included:
+                print "Skipping test '%s'.." % (fullcmd)
+                continue
+
+            #a.extend(["-o", "/tmp/xx" + a[0] + ".xml"])
+            #print a
+            #a = ["ccdash.py"].extend(a)
+            b = ["ccdash.py"]
+            b.extend(a)
+            a = b
+            #print a
+            ccdash.main(a)
+            counter = counter + 1
+        self.post_action()
+
+
+#
+# GNU test configurator
+#
+class GNUTestBuilder(TestBuilder):
+    def __init__(self, config, build_config_name="", user_mak="", \
+                 config_site="", cross_compile="", exclude=[], not_exclude=[]):
+        TestBuilder.__init__(self, config, build_config_name=build_config_name,
+                             user_mak=user_mak, config_site=config_site,
+                             exclude=exclude, not_exclude=not_exclude)
+        self.cross_compile = cross_compile
+        if self.cross_compile and self.cross_compile[-1] != '-':
+            self.cross_compile.append("-")
+
+    def build_tests(self):
+        if self.cross_compile:
+            suffix = self.cross_compile
+            build_name =  suffix + gcc_version(self.cross_compile + "gcc")
+        else:
+            proc = subprocess.Popen(self.config.base_dir+"/config.guess",
+                                    stdout=subprocess.PIPE)
+            suffix = proc.stdout.readline().rstrip(" \r\n")
+            build_name =  suffix+"-"+gcc_version(self.cross_compile + "gcc")
+
+        if self.build_config_name:
+            build_name = build_name + "-" + self.build_config_name
+        cmds = []
+        cmds.extend(update_ops)
+        cmds.extend(gnu_build_ops)
+        cmds.extend(std_test_ops)
+        cmds.extend(build_pjsua_test_ops())
+        self.ccdash_args = []
+        for c in cmds:
+            c.cmdline = c.cmdline.replace("$SUFFIX", suffix)
+            args = c.encode(self.config.base_dir)
+            args.extend(["-U", self.config.url, 
+                         "-S", self.config.site, 
+                         "-T", self.stamp(), 
+                         "-B", build_name, 
+                         "-G", self.config.group])
+            args.extend(self.config.options)
+            self.ccdash_args.append(args)
+
+
diff --git a/tests/cdash/cfg_gnu.py b/tests/cdash/cfg_gnu.py
new file mode 100644
index 00000000..55bb82bc
--- /dev/null
+++ b/tests/cdash/cfg_gnu.py
@@ -0,0 +1,34 @@
+import builder
+import os
+import sys
+
+# Each configurator must export this function
+def create_builder(args):
+    # (optional) args format:
+    #   site configuration module. If not specified, "cfg_site" is implied
+
+    if len(args)>0:
+        file = args[0]
+    else:
+        file = "cfg_site"
+
+    if os.access(file+".py", os.F_OK) == False:
+	print "Error: file '%s.py' doesn't exist." % (file)
+	sys.exit(1)
+
+    cfg_site = __import__(file)
+    test_cfg = builder.BaseConfig(cfg_site.BASE_DIR, \
+                                  cfg_site.URL, \
+                                  cfg_site.SITE_NAME, \
+                                  cfg_site.GROUP, \
+                                  cfg_site.OPTIONS)
+
+    builders = [
+        builder.GNUTestBuilder(test_cfg, build_config_name="default",
+                               user_mak="export CFLAGS+=-Wall\n",
+                               config_site="#define PJ_TODO(x)\n",
+                               exclude=cfg_site.EXCLUDE,
+                               not_exclude=cfg_site.NOT_EXCLUDE)
+        ]
+
+    return builders
diff --git a/tests/cdash/cfg_site_sample.py b/tests/cdash/cfg_site_sample.py
new file mode 100644
index 00000000..147aa496
--- /dev/null
+++ b/tests/cdash/cfg_site_sample.py
@@ -0,0 +1,25 @@
+import builder
+
+# Your site name
+SITE_NAME="Newham2"
+
+# The URL where tests will be submitted to
+URL = "http://192.168.0.2/dash/submit.php?project=PJSIP"
+
+# Test group
+GROUP = "Experimental"
+
+# PJSIP base directory
+BASE_DIR = "/root/project/pjproject"
+
+# List of additional ccdash options
+#OPTIONS = ["-o", "out.xml", "-y"]
+OPTIONS = ["-o", "out.xml"]
+
+# List of regular expression of test patterns to be excluded
+EXCLUDE = [".*"]
+
+# List of regular expression of test patterns to be included (even
+# if they match EXCLUDE patterns)
+NOT_EXCLUDE = ["run.py mod_run.*100_simple"]
+#"configure", "update", "build.*make", "build", "run.py mod_run.*100_simple"]
diff --git a/tests/cdash/inc_test.py b/tests/cdash/inc_test.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/tests/cdash/inc_test.py
@@ -0,0 +1 @@
+
diff --git a/tests/cdash/main.py b/tests/cdash/main.py
new file mode 100644
index 00000000..9f707bd7
--- /dev/null
+++ b/tests/cdash/main.py
@@ -0,0 +1,20 @@
+import sys
+
+if len(sys.argv)==1:
+    print "Usage: main.py cfg_file [cfg_site]"
+    print "Example:"
+    print "  main.py cfg_gnu"
+    print "  main.py cfg_gnu custom_cfg_site"
+    sys.exit(1)
+
+
+args = []
+args.extend(sys.argv)
+args.remove(args[1])
+args.remove(args[0])
+
+cfg_file = __import__(sys.argv[1])
+builders = cfg_file.create_builder(args)
+
+for builder in builders:
+    builder.execute()