Ticket #543:

 - added options to run.py
 - passing options in runall.py to run.py
 - removing userdata in module callback functions




git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@2078 74dad513-b988-da41-8d7b-12977e46ad98

9 files changed, 184 insertions, 101 deletions

diff --git a/pjsip-apps/src/test-pjsua/inc_cfg.py b/pjsip-apps/src/test-pjsua/inc_cfg.py
index f20a9349..3c601079 100644
--- a/pjsip-apps/src/test-pjsua/inc_cfg.py
+++ b/pjsip-apps/src/test-pjsua/inc_cfg.py
@@ -1,11 +1,14 @@
 # $Id$
 import random
-from config_site import *
 
 DEFAULT_ECHO = True
 DEFAULT_TRACE = True
 DEFAULT_START_SIP_PORT = 50000
 
+# Shared vars
+ARGS = []		# arguments containing script module & config
+HAS_SND_DEV = 1		# specify 1 if system has sound device and prefer to use sound device in the tests
+
 # Individual pjsua instance configuration class
 class InstanceParam:
 	# Name to identify this pjsua instance (e.g. "caller", "callee", etc.)
@@ -75,20 +78,17 @@ class TestParam:
 	# the function for test body
 	test_func = None
 	post_func = None
-	user_data = None
 	def __init__(	self, 
 			title, 		# Test title
 			inst_params, 	# InstanceParam's as list
 			func=None,
 			skip=False,
-			post_func=None,
-			user_data=None):
+			post_func=None):
 		self.title = title
 		self.inst_params = inst_params
 		self.skip = skip
 		self.test_func = func
 		self.post_func = post_func
-		self.user_data = user_data
 
 
 ###################################
diff --git a/pjsip-apps/src/test-pjsua/mod_call.py b/pjsip-apps/src/test-pjsua/mod_call.py
index c3bb09c0..7081aed4 100644
--- a/pjsip-apps/src/test-pjsua/mod_call.py
+++ b/pjsip-apps/src/test-pjsua/mod_call.py
@@ -6,7 +6,7 @@ import inc_const as const
 from inc_cfg import *
 
 # Load configuration
-cfg_file = imp.load_source("cfg_file", sys.argv[2])
+cfg_file = imp.load_source("cfg_file", ARGS[1])
 
 # Check media flow between ua1 and ua2
 def check_media(ua1, ua2):
@@ -20,7 +20,7 @@ def check_media(ua1, ua2):
 
 
 # Test body function
-def test_func(t, user_data):
+def test_func(t):
 	callee = t.process[0]
 	caller = t.process[1]
 
diff --git a/pjsip-apps/src/test-pjsua/mod_media_playrec.py b/pjsip-apps/src/test-pjsua/mod_media_playrec.py
index 2fc25703..bd064e92 100644
--- a/pjsip-apps/src/test-pjsua/mod_media_playrec.py
+++ b/pjsip-apps/src/test-pjsua/mod_media_playrec.py
@@ -14,9 +14,10 @@ import sys
 import re
 import subprocess
 import inc_const as const
+from inc_cfg import *
 
 # Load configuration
-cfg_file = imp.load_source("cfg_file", sys.argv[2])
+cfg_file = imp.load_source("cfg_file", ARGS[1])
 
 # WAV similarity calculator
 COMPARE_WAV_EXE = "tools/cmp_wav.exe"
@@ -24,13 +25,15 @@ COMPARE_WAV_EXE = "tools/cmp_wav.exe"
 # Threshold to declare degradation is too high when result is lower than this value
 COMPARE_THRESHOLD = 2
 
-# UserData
-class mod_media_playrec_user_data:
-	input_filename = ""
-	output_filename = ""
+# COMPARE params
+input_filename	= ""			# Input filename
+output_filename = ""			# Output filename
 
 # Test body function
 def test_func(t, ud):
+	global input_filename
+	global output_filename
+
 	endpt = t.process[0]
 	
 	# Get input file name
@@ -67,6 +70,9 @@ def test_func(t, ud):
 
 # Post body function
 def post_func(t, ud):
+	global input_filename
+	global output_filename
+
 	endpt = t.process[0]
 
 	# Check WAV similarity
@@ -93,4 +99,3 @@ def post_func(t, ud):
 test = cfg_file.test_param
 test.test_func = test_func
 test.post_func = post_func
-test.user_data = mod_media_playrec_user_data()
diff --git a/pjsip-apps/src/test-pjsua/mod_pesq.py b/pjsip-apps/src/test-pjsua/mod_pesq.py
index 9853a756..de92eb12 100644
--- a/pjsip-apps/src/test-pjsua/mod_pesq.py
+++ b/pjsip-apps/src/test-pjsua/mod_pesq.py
@@ -21,36 +21,35 @@ import inc_const as const
 from inc_cfg import *
 
 # Load configuration
-cfg_file = imp.load_source("cfg_file", sys.argv[2])
+cfg_file = imp.load_source("cfg_file", ARGS[1])
 
 # PESQ configs
-# PESQ_THRESHOLD specifies the minimum acceptable PESQ MOS value, so test can be declared successful
-PESQ = "tools/pesq.exe"
-PESQ_DEFAULT_THRESHOLD = 3.4
-
-# UserData
-class mod_pesq_user_data:
-	# Sample rate option for PESQ
-	pesq_sample_rate_opt = ""
-	# Input/Reference filename
-	input_filename = ""
-	# Output/Degraded filename
-	output_filename = ""
+PESQ = "tools/pesq.exe"			# PESQ executable path
+PESQ_DEFAULT_THRESHOLD = 3.4		# Default minimum acceptable PESQ MOS value
+
+# PESQ params
+pesq_sample_rate_opt = ""		# Sample rate option for PESQ
+input_filename	= ""			# Input/Reference filename
+output_filename = ""			# Output/Degraded filename
+
 
 # Test body function
-def test_func(t, user_data):
+def test_func(t):
+	global pesq_sample_rate_opt
+	global input_filename
+	global output_filename
 
 	ua1 = t.process[0]
 	ua2 = t.process[1]
 
 	# Get input file name
-	user_data.input_filename = re.compile(const.MEDIA_PLAY_FILE).search(ua1.inst_param.arg).group(1)
+	input_filename = re.compile(const.MEDIA_PLAY_FILE).search(ua1.inst_param.arg).group(1)
 
 	# Get output file name
-	user_data.output_filename = re.compile(const.MEDIA_REC_FILE).search(ua2.inst_param.arg).group(1)
+	output_filename = re.compile(const.MEDIA_REC_FILE).search(ua2.inst_param.arg).group(1)
 
 	# Get WAV input length, in seconds
-	fin = wave.open(user_data.input_filename, "r")
+	fin = wave.open(input_filename, "r")
 	if fin == None:
 		raise TestError("Failed opening input WAV file")
 	inwavlen = fin.getnframes() * 1.0 / fin.getframerate()
@@ -59,7 +58,7 @@ def test_func(t, user_data):
 	print "WAV input len = " + str(inwavlen) + "s"
 
 	# Get clock rate of the output
-	mo_clock_rate = re.compile("\.(\d+)\.wav").search(user_data.output_filename)
+	mo_clock_rate = re.compile("\.(\d+)\.wav").search(output_filename)
 	if (mo_clock_rate==None):
 		raise TestError("Cannot compare input & output, incorrect output filename format")
 	clock_rate = mo_clock_rate.group(1)
@@ -72,18 +71,16 @@ def test_func(t, user_data):
 	# Get matched input file from output file
 	# (PESQ evaluates only files whose same clock rate & channel count)
 	if channel_count == 2:
-	    if re.search("\.\d+\.\d+\.wav", user_data.input_filename) != None:
-		    user_data.input_filename = re.sub("\.\d+\.\d+\.wav", 
-						      "." + str(channel_count) + "."+clock_rate+".wav", user_data.input_filename)
+	    if re.search("\.\d+\.\d+\.wav", input_filename) != None:
+		    input_filename = re.sub("\.\d+\.\d+\.wav", "." + str(channel_count) + "."+clock_rate+".wav", input_filename)
 	    else:
-		    user_data.input_filename = re.sub("\.\d+\.wav", 
-						      "." + str(channel_count) + "."+clock_rate+".wav", user_data.input_filename)
+		    input_filename = re.sub("\.\d+\.wav", "." + str(channel_count) + "."+clock_rate+".wav", input_filename)
 
 	if (clock_rate != "8") & (clock_rate != "16"):
 		raise TestError("PESQ only works on clock rate 8kHz or 16kHz, clock rate used = "+clock_rate+ "kHz")
 
 	# Get conference clock rate of UA2 for PESQ sample rate option
-	user_data.pesq_sample_rate_opt = "+" + clock_rate + "000"
+	pesq_sample_rate_opt = "+" + clock_rate + "000"
 
 	# UA1 making call
 	ua1.send("m")
@@ -113,11 +110,15 @@ def test_func(t, user_data):
 
 
 # Post body function
-def post_func(t, user_data):
+def post_func(t):
+	global pesq_sample_rate_opt
+	global input_filename
+	global output_filename
+
 	endpt = t.process[0]
 
 	# Execute PESQ
-	fullcmd = PESQ + " " + user_data.pesq_sample_rate_opt + " " + user_data.input_filename + " " + user_data.output_filename
+	fullcmd = PESQ + " " + pesq_sample_rate_opt + " " + input_filename + " " + output_filename
 	endpt.trace("Popen " + fullcmd)
 	pesq_proc = subprocess.Popen(fullcmd, stdout=subprocess.PIPE, universal_newlines=True)
 	pesq_out  = pesq_proc.communicate()
@@ -146,5 +147,4 @@ def post_func(t, user_data):
 test = cfg_file.test_param
 test.test_func = test_func
 test.post_func = post_func
-test.user_data = mod_pesq_user_data()
 
diff --git a/pjsip-apps/src/test-pjsua/mod_pres.py b/pjsip-apps/src/test-pjsua/mod_pres.py
index 7b00d8e4..7dafd52f 100644
--- a/pjsip-apps/src/test-pjsua/mod_pres.py
+++ b/pjsip-apps/src/test-pjsua/mod_pres.py
@@ -3,13 +3,14 @@ import time
 import imp
 import sys
 import inc_const as const
+from inc_cfg import *
 
 # Load configuration
-cfg_file = imp.load_source("cfg_file", sys.argv[2])
+cfg_file = imp.load_source("cfg_file", ARGS[1])
 
 
 # Test body function
-def test_func(t, user_data):
+def test_func(t):
 	u1 = t.process[0]
 	uri1 = cfg_file.test_param.inst_params[0].uri
 	acc1 = "-1"
diff --git a/pjsip-apps/src/test-pjsua/mod_run.py b/pjsip-apps/src/test-pjsua/mod_run.py
index 379b4f26..03548eea 100644
--- a/pjsip-apps/src/test-pjsua/mod_run.py
+++ b/pjsip-apps/src/test-pjsua/mod_run.py
@@ -1,10 +1,11 @@
-# $Id:$
+# $Id$
 import imp
 import sys
 
+from inc_cfg import *
 
 # Read configuration
-cfg_file = imp.load_source("cfg_file", sys.argv[2])
+cfg_file = imp.load_source("cfg_file", ARGS[1])
 
 # Here where it all comes together
 test = cfg_file.test_param
diff --git a/pjsip-apps/src/test-pjsua/mod_sendto.py b/pjsip-apps/src/test-pjsua/mod_sendto.py
index 7b1f4823..826c509f 100644
--- a/pjsip-apps/src/test-pjsua/mod_sendto.py
+++ b/pjsip-apps/src/test-pjsua/mod_sendto.py
@@ -7,10 +7,10 @@ import re
 from inc_cfg import *
 
 # Read configuration
-cfg_file = imp.load_source("cfg_file", sys.argv[2])
+cfg_file = imp.load_source("cfg_file", ARGS[1])
 
 # Test body function
-def test_func(t, userdata):
+def test_func(t):
 	pjsua = t.process[0]
 	# Create dialog
 	dlg = sip.Dialog("127.0.0.1", pjsua.inst_param.sip_port, 
diff --git a/pjsip-apps/src/test-pjsua/run.py b/pjsip-apps/src/test-pjsua/run.py
index c1947efa..7b2d3a9f 100644
--- a/pjsip-apps/src/test-pjsua/run.py
+++ b/pjsip-apps/src/test-pjsua/run.py
@@ -6,39 +6,87 @@ import os
 import subprocess
 import random
 import time
+import getopt
 
 import inc_const as const
-from inc_cfg import *
+import inc_cfg as inc
+
+# Vars
+G_EXE = ""		# pjsua executable path
+G_INUNIX = False	# flags that test is running in Unix
+
+
+# Usage string
+usage = \
+"""
+run.py - Automated test driver
+
+Usage:
+	run.py [options] MODULE CONFIG
+Options:
+	--exe, -e		pjsua executable path
+	--null-audio, -n	use null audio
+Sample:
+	run.py -n mod_run.py scripts-run/100_simple.py
+"""
+
+# Parse arguments
+try:
+    opts, args = getopt.getopt(sys.argv[1:], "hne:", ["help", "null-audio", "exe="])
+except getopt.GetoptError, err:
+    print str(err)
+    print usage
+    sys.exit(2)
+for o, a in opts:
+    if o in ("-h", "--help"):
+	print usage
+	sys.exit()
+    elif o in ("-n", "--null-audio"):
+        inc.HAS_SND_DEV = 0
+    elif o in ("-e", "--exe"):
+        G_EXE = a
+    else:
+        print "Unknown options"
+	sys.exit(2)
+
+if len(args) != 2:
+	print "Invalid arguments"
+	print usage
+	sys.exit(2)
+
+# Set global ARGS to be used by modules
+inc.ARGS = args
 
 # Get the pjsua executable name
-if sys.platform.find("win32")!=-1:
-    e = "../../bin/pjsua_vc6d.exe"
-    st1 = os.stat(e)
-    if st1 != None:
-	G_EXE = e
-    e = "../../bin/pjsua_vc6d.exe"
-    st2 = os.stat(e)
-    if st2 != None and st2.st_mtime > st1.st_mtime:
-	G_EXE = e
-	st1 = st2
-    if G_EXE=="":
-	print "Unable to find valid pjsua. Please build pjsip first"
-	sys.exit(1)
-    G_INUNIX = False
-else:
-    f = open("../../../build.mak", "r")
-    while True:
-	line = f.readline()
-	if not line:
-	    break
-	if line.find("TARGET_NAME")!=-1:
-	    print line
-	    G_EXE="../../bin/pjsua-" + line.split(":= ")[1]
-	    break
-    if G_EXE=="":
-	print "Unable to find ../../../build.mak. Please build pjsip first"
-	sys.exit(1)
-    G_INUNIX = True
+if G_EXE == "":
+	if sys.platform.find("win32")!=-1:
+	    e = "../../bin/pjsua_vc6d.exe"
+	    st1 = os.stat(e)
+	    if st1 != None:
+		G_EXE = e
+	    e = "../../bin/pjsua_vc6d.exe"
+	    st2 = os.stat(e)
+	    if st2 != None and st2.st_mtime > st1.st_mtime:
+		G_EXE = e
+		st1 = st2
+	    if G_EXE=="":
+		print "Unable to find valid pjsua. Please build pjsip first"
+		sys.exit(1)
+	    G_INUNIX = False
+	else:
+	    f = open("../../../build.mak", "r")
+	    while True:
+		line = f.readline()
+		if not line:
+		    break
+		if line.find("TARGET_NAME")!=-1:
+		    print line
+		    G_EXE="../../bin/pjsua-" + line.split(":= ")[1]
+		    break
+	    if G_EXE=="":
+		print "Unable to find ../../../build.mak. Please build pjsip first"
+		sys.exit(1)
+	    G_INUNIX = True
 
 
 G_EXE = G_EXE.rstrip("\n\r \t")
@@ -74,14 +122,14 @@ class Expect:
 		while True:
 			line = self.proc.stdout.readline()
 		  	if line == "":
-				raise TestError(self.name + ": Premature EOF")
+				raise inc.TestError(self.name + ": Premature EOF")
 			# Print the line if echo is ON
 			if self.echo:
 				print self.name + ": " + line,
 			# Trap assertion error
 			if self.ra.search(line) != None:
 				if raise_on_error:
-					raise TestError(self.name + ": " + line)
+					raise inc.TestError(self.name + ": " + line)
 				else:
 					return None
 			# Count stdout refresh text. 
@@ -90,7 +138,7 @@ class Expect:
 				if refresh_cnt >= 6:
 					self.trace("Timed-out!")
 					if raise_on_error:
-						raise TestError(self.name + " " + title + ": Timeout expecting pattern: \"" + pattern + "\"")
+						raise inc.TestError(self.name + " " + title + ": Timeout expecting pattern: \"" + pattern + "\"")
 					else:
 						return None		# timeout
 			# Search for expected text
@@ -130,15 +178,8 @@ def handle_error(errmsg, t, close_processes = True):
 #########################
 # MAIN	
 
-if len(sys.argv)!=3:
-	print "Usage: run.py MODULE CONFIG"
-	print "Sample:"
-	print "  run.py mod_run.py scripts-run/100_simple.py"
-	sys.exit(1)
-
-
 # Import the test script
-script = imp.load_source("script", sys.argv[1])  
+script = imp.load_source("script", inc.ARGS[0])  
 
 # Init random seed
 random.seed()
@@ -176,14 +217,14 @@ for inst_param in script.test.inst_params:
 		# add running instance
 		script.test.process.append(p)
 
-	except TestError, e:
+	except inc.TestError, e:
 		handle_error(e.desc, script.test)
 
 # Run the test function
 if script.test.test_func != None:
 	try:
-		script.test.test_func(script.test, script.test.user_data)
-	except TestError, e:
+		script.test.test_func(script.test)
+	except inc.TestError, e:
 		handle_error(e.desc, script.test)
 
 # Shutdown all instances
@@ -203,8 +244,8 @@ for p in script.test.process:
 # Run the post test function
 if script.test.post_func != None:
 	try:
-		script.test.post_func(script.test, script.test.user_data)
-	except TestError, e:
+		script.test.post_func(script.test)
+	except inc.TestError, e:
 		handle_error(e.desc, script.test, False)
 
 # Done
diff --git a/pjsip-apps/src/test-pjsua/runall.py b/pjsip-apps/src/test-pjsua/runall.py
index 60ad9dc1..b374c549 100644
--- a/pjsip-apps/src/test-pjsua/runall.py
+++ b/pjsip-apps/src/test-pjsua/runall.py
@@ -2,6 +2,9 @@
 import os
 import sys
 import time
+import re
+import shutil
+
 
 # Usage:
 #  runall.py [test-to-resume]
@@ -53,23 +56,48 @@ for pat in excluded_tests:
 # Resume test?
 resume_script=""
 if len(sys.argv) > 1:
-    if sys.argv[1][0]=='-' or sys.argv[1][0]=='/':
+    if sys.argv[1]=='-r' or sys.argv[1]=='--resume':
+	resume_script=sys.argv[2]
+    if sys.argv[1]=='/h' or sys.argv[1]=='-h' or sys.argv[1]=='--help' or sys.argv[1]=='/help':
         print "Usage:"
-	print "  runall.py [RESUME]"
-	print "where"
-	print "  RESUME is string/substring to specify where to resume tests."
-	print "  If this argument is omited, tests will start from the beginning."
+	print "  runall.py [OPTIONS] [run.py OPTIONS]"
+	print "Options:"
+	print "  --resume,-r RESUME"
+	print "  where"
+	print "      RESUME is string/substring to specify where to resume tests."
+	print "      If this argument is omited, tests will start from the beginning."
 	sys.exit(0)
-    resume_script=sys.argv[1]
 
 
+# Generate arguments for run.py
+argv = sys.argv
+argv_to_skip = 1
+if resume_script != "":
+    argv_to_skip += 2
+argv_st = ""
+for a in argv:
+    if argv_to_skip > 0:
+        argv_to_skip -= 1
+    else:
+        argv_st += a + " "
+
+
+# Init vars
+failed_cnt = 0
+
+# Create "logs" directory
+try:
+    os.mkdir("logs")
+except:
+    print
+
 # Now run the tests
 for t in tests:
 	if resume_script!="" and t.find(resume_script)==-1:
 	    print "Skipping " + t +".."
 	    continue
 	resume_script=""
-	cmdline = "python run.py " + t
+	cmdline = "python run.py " + argv_st + t
 	t0 = time.time()
 	print "Running " + cmdline + "...",
 	ret = os.system(cmdline + " > output.log")
@@ -77,10 +105,17 @@ for t in tests:
 	if ret != 0:
 		dur = int(t1 - t0)
 		print " failed!! [" + str(dur) + "s]"
-		print "Please see 'output.log' for the test log."
-		sys.exit(1)
+		logname = re.search(".*\s+(.*)", t).group(1)
+		logname = re.sub("[\\\/]", "_", logname)
+		logname = "logs/" + logname
+		shutil.move("output.log", logname)
+		print "Please see '" + logname + "' for the test log."
 	else:
 		dur = int(t1 - t0)
 		print " ok [" + str(dur) + "s]"
 
-print "All tests completed successfully"
+if failed_cnt == 0:
+	print "All tests completed successfully"
+else:
+	print "Tests completed, with " +  str(failed_cnt) + " test(s) failed"
+