Ticket #543: initial framework for pjsua testing, still needs lots of test scenarios

git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@2009 74dad513-b988-da41-8d7b-12977e46ad98

12 files changed, 465 insertions, 0 deletions

diff --git a/pjsip-apps/src/test-pjsua/README.TXT b/pjsip-apps/src/test-pjsua/README.TXT
new file mode 100644
index 00000000..fed4be79
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/README.TXT
@@ -0,0 +1,34 @@
+

+                           PJSUA TEST FRAMEWORK

+                        =========================

+

+0. What is this

+---------------

+This is the automated testing scripts for pjsua. It can do many things (just don't ask it to write good documentation :) ).

+

+

+1. Requirements

+---------------

+To run the tests you need:

+ - Python (tested with Python 2.5.2)

+ - pjsua application, built and placed in pjsip-apps/bin directory

+ - the pjsua must be built with:

+     - TLS enabled

+     - SRTP enabled

+

+

+2. Using

+--------

+To run all the tests:

+  $ python runall.py

+

+To run individual test:

+  $ python run.py MODULE CONFIG

+

+The run.py is the main entry for the test. It imports the various inc_xxx.py files, and it will load the MODULE. The MODULE contains specific test flows, and we have couple of them:

+  - mod_run.py: this module implements a simple test which just run pjsua with the configuration from CONFIG file and checks if pjsua can start properly.

+  - mod_call.py: this module implements call testing where it spawns two pjsua instances each with configurations as specified in CONFIG file, makes one pjsua call the other, and checks if the call can be established.

+

+Example:

+  $ python run.py mod_run.py scripts-run/100_simple.py

+  $ python run.py mod_call.py scripts-call/100_simple.py

diff --git a/pjsip-apps/src/test-pjsua/inc_cfg.py b/pjsip-apps/src/test-pjsua/inc_cfg.py
new file mode 100644
index 00000000..cf5f6117
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/inc_cfg.py
@@ -0,0 +1,25 @@
+# $Id$

+

+DEFAULT_ECHO = True

+DEFAULT_TRACE = True

+

+# Individual pjsua config class

+class Config:

+	# pjsua command line arguments, concatenated in string

+	arg = ""

+	# Specify whether pjsua output should be echoed to stdout

+	echo_enabled = DEFAULT_ECHO

+	# Enable/disable test tracing

+	trace_enabled = DEFAULT_TRACE

+	def __init__(self, arg, echo_enabled=DEFAULT_ECHO, trace_enabled=DEFAULT_TRACE):

+		self.arg = arg

+		self.echo_enabled = echo_enabled

+		self.trace_enabled = trace_enabled

+

+# Call config class

+class CallConfig:

+	def __init__(self, title, callee_cfg, caller_cfg):

+		self.title = title

+		self.callee_cfg = callee_cfg

+		self.caller_cfg = caller_cfg

+

diff --git a/pjsip-apps/src/test-pjsua/inc_const.py b/pjsip-apps/src/test-pjsua/inc_const.py
new file mode 100644
index 00000000..b7022343
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/inc_const.py
@@ -0,0 +1,49 @@
+# $Id$

+# Useful constants

+

+

+##########################

+# MENU OUTPUT

+#

+

+

+##########################

+# EVENTS

+#

+

+# Text to expect when there is incoming call

+EVENT_INCOMING_CALL = "Press .* answer"

+

+

+##########################

+# CALL STATES

+#

+

+# Call state is CALLING

+STATE_CALLING = "state.*CALLING"

+# Call state is CONFIRMED

+STATE_CONFIRMED = "state.*CONFIRMED"

+# Call state is DISCONNECTED

+STATE_DISCONNECTED = "Call .* DISCONNECTED"

+

+# Media call is put on-hold

+MEDIA_HOLD = "Media for call [0-9]+ is suspended.*hold"

+# Media call is active

+MEDIA_ACTIVE = "Media for call [0-9]+ is active"

+# RX_DTMF

+RX_DTMF = "Incoming DTMF on call [0-9]+: "

+

+##########################

+# MISC

+#

+

+# The command prompt

+PROMPT = ">>>"

+# When pjsua has been destroyed

+DESTROYED = "PJSUA destroyed"

+# Assertion failure

+ASSERT = "Assertion failed"

+# Stdout refresh text

+STDOUT_REFRESH = "XXSTDOUT_REFRESHXX"

+

+

diff --git a/pjsip-apps/src/test-pjsua/inc_param.py b/pjsip-apps/src/test-pjsua/inc_param.py
new file mode 100644
index 00000000..e897d140
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/inc_param.py
@@ -0,0 +1,46 @@
+# $Id$

+###########################################

+# pjsua instantiation parameter

+class Pjsua:

+	# instance name

+	name = ""

+	# command line arguments. Default is empty.

+	# sample:

+	#  args = "--null-audio --local-port 0"

+	args = ""

+	# list containing send/expect/title list. Default empty.

+	# The inside list contains three items, all are optional:

+	#  - the command to be sent to pjsua menu

+	#  - the string to expect

+	#  - optional string to describe what this is doing

+	# Sample of command list containing two list items:

+	#  cmds = [["sleep 50",""], ["q","", "quitting.."]]

+	cmds = []

+	# print out the stdout output of this pjsua?

+	echo = False

+	# print out commands interacting with this pjsua?

+	trace = False

+	def __init__(self, name, args="", echo=False, trace=False, cmds=[]):

+		self.name = name

+		self.args = args

+		self.echo = echo

+		self.trace = trace

+		self.cmds = cmds

+			

+############################################

+# Test parameter class

+class Test:

+	title = ""

+	# params is list containing PjsuaParam objects

+	run = []

+	# list of Expect instances, to be filled at run-time by

+        # the test program	

+	process = []

+	# the function for test body

+	test_func = None

+	def __init__(self, title, run, func=None):

+		self.title = title

+		self.run = run

+		self.test_func = func

+

+

diff --git a/pjsip-apps/src/test-pjsua/mod_call.py b/pjsip-apps/src/test-pjsua/mod_call.py
new file mode 100644
index 00000000..2cd81951
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/mod_call.py
@@ -0,0 +1,92 @@
+# $Id$

+import time

+import imp

+import sys

+import inc_param as param

+import inc_const as const

+

+# Load configuration

+cfg_file = imp.load_source("cfg_file", sys.argv[2])

+

+# Test title

+title = cfg_file.config.title

+port1 = "9060"

+

+# First pjsua

+p1 = param.Pjsua(

+		"callee",

+		args = cfg_file.config.callee_cfg.arg + " --local-port="+port1,

+		echo = cfg_file.config.callee_cfg.echo_enabled,

+		trace = cfg_file.config.callee_cfg.trace_enabled

+		)

+

+# Second pjsua, make call to the first one

+p2 = param.Pjsua(

+		"caller",

+		args = cfg_file.config.caller_cfg.arg + " --local-port=0",

+		echo = cfg_file.config.caller_cfg.echo_enabled,

+		trace = cfg_file.config.caller_cfg.trace_enabled

+		)

+

+# Test body function

+def test_func(t):

+	callee = t.process[0]

+	caller = t.process[1]

+

+	# Caller making call

+	caller.send("m")

+	caller.send("sip:localhost:" + port1)

+	caller.expect(const.STATE_CALLING)

+	

+	# Callee answers with 200/OK

+	time.sleep(1)

+	callee.expect(const.EVENT_INCOMING_CALL)

+	callee.send("a")

+	callee.send("200")

+

+	# Wait until call is connected in both endpoints

+	time.sleep(1)

+	if callee.expect(const.STATE_CONFIRMED, False)==None:

+		raise TestError("Call failed")

+	caller.expect(const.STATE_CONFIRMED)

+

+	# Hold call

+	time.sleep(2)

+	caller.send("H")

+	caller.expect(const.MEDIA_HOLD)

+	callee.expect(const.MEDIA_HOLD)

+	

+	# Release hold

+	time.sleep(2)

+	caller.send("v")

+	caller.expect(const.MEDIA_ACTIVE)

+	callee.expect(const.MEDIA_ACTIVE)

+

+	# UPDATE

+	time.sleep(2)

+	callee.send("U")

+	callee.expect(const.MEDIA_ACTIVE)

+	caller.expect(const.MEDIA_ACTIVE)

+	

+	# Send DTMF

+	time.sleep(2)

+	caller.send("#")

+	caller.send("1122")

+	callee.expect(const.RX_DTMF + "1")

+	callee.expect(const.RX_DTMF + "1")

+	callee.expect(const.RX_DTMF + "2")

+	callee.expect(const.RX_DTMF + "2")

+

+	# Hangup call

+	time.sleep(1)

+	caller.send("h")

+

+	# Wait until calls are cleared in both endpoints

+	caller.expect(const.STATE_DISCONNECTED)

+	callee.expect(const.STATE_DISCONNECTED)

+	

+

+# Here where it all comes together

+test = param.Test(title, run=[p1, p2], func=test_func)

+

+

diff --git a/pjsip-apps/src/test-pjsua/mod_run.py b/pjsip-apps/src/test-pjsua/mod_run.py
new file mode 100644
index 00000000..c5e475d3
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/mod_run.py
@@ -0,0 +1,19 @@
+# $Id$

+import imp

+import sys

+import inc_param as param

+

+

+# Read configuration

+cfg_file = imp.load_source("cfg_file", sys.argv[2])

+

+# Test title

+title = "Basic pjsua"

+

+# Param to spawn pjsua

+p1 = param.Pjsua("pjsua", args=cfg_file.config.arg, 

+		 echo=cfg_file.config.echo_enabled, 

+		 trace=cfg_file.config.trace_enabled)

+

+# Here where it all comes together

+test = param.Test(title, run=[p1])

diff --git a/pjsip-apps/src/test-pjsua/run.py b/pjsip-apps/src/test-pjsua/run.py
new file mode 100644
index 00000000..16d23ca2
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/run.py
@@ -0,0 +1,152 @@
+# $Id$

+import sys

+import imp

+import re

+import subprocess

+import time

+

+import inc_param as param

+import inc_const as const

+

+# Defaults

+G_ECHO=True

+G_TRACE=False

+G_EXE="..\\..\\bin\\pjsua_vc6d.exe"

+

+###################################

+# TestError exception

+class TestError:

+	desc = ""

+	def __init__(self, desc):

+		self.desc = desc

+

+###################################

+# Poor man's 'expect'-like class

+class Expect:

+	proc = None

+	echo = False

+	trace_enabled = False

+	name = ""

+	rh = re.compile(const.DESTROYED)

+	ra = re.compile(const.ASSERT, re.I)

+	rr = re.compile(const.STDOUT_REFRESH)

+	def __init__(self, name, exe, args="", echo=G_ECHO, trace_enabled=G_TRACE):

+		self.name = name

+		self.echo = echo

+		self.trace_enabled = trace_enabled

+		fullcmd = exe + " " + args + " --stdout-refresh=5 --stdout-refresh-text=" + const.STDOUT_REFRESH

+		self.trace("Popen " + fullcmd)

+		self.proc = subprocess.Popen(fullcmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)

+	def send(self, cmd):

+		self.trace("send " + cmd)

+		self.proc.stdin.writelines(cmd + "\n")

+	def expect(self, pattern, raise_on_error=True):

+		self.trace("expect " + pattern)

+		r = re.compile(pattern, re.I)

+		refresh_cnt = 0

+		while True:

+			line = self.proc.stdout.readline()

+		  	if line == "":

+				raise TestError("Premature EOF")

+			# Print the line if echo is ON

+			if self.echo:

+				print self.name + ": " + line,

+			# Trap assertion error

+			if self.ra.search(line) != None:

+				raise TestError(line)

+			# Count stdout refresh text. 

+			if self.rr.search(line) != None:

+				refresh_cnt = refresh_cnt+1

+				if refresh_cnt >= 6:

+					self.trace("Timed-out!")

+					if raise_on_error:

+						raise TestError("Timeout expecting pattern: " + pattern)

+					else:

+						return None		# timeout

+			# Search for expected text

+			if r.search(line) != None:

+				return line

+	def wait(self):

+		self.trace("wait")

+		self.proc.wait()

+	def trace(self, s):

+		if self.trace_enabled:

+			print self.name + ": " + "====== " + s + " ======"

+

+#########################

+# Error handling

+def handle_error(errmsg, t):

+	print "====== Caught error: " + errmsg + " ======"

+	time.sleep(1)

+	for p in t.process:

+		p.send("q")

+		p.expect(const.DESTROYED)

+		p.wait()

+	print "Test completed with error: " + errmsg

+	sys.exit(1)

+

+

+#########################

+# MAIN	

+

+if len(sys.argv)!=3:

+	print "Usage: run.py MODULE CONFIG"

+	print "Sample:"

+	print "  run.py mod_run.py scripts-run/100_simple.py"

+	sys.exit(1)

+

+

+# Import the test script

+script = imp.load_source("script", sys.argv[1])  

+

+# Validate

+if script.test == None:

+	print "Error: no test defined"

+	sys.exit(1)

+

+if len(script.test.run) == 0:

+	print "Error: test doesn't contain pjsua run descriptions"

+	sys.exit(1)

+

+# Instantiate pjsuas

+print "====== Running " + script.test.title + " ======"

+for run in script.test.run:

+	try:

+		p = Expect(run.name, G_EXE, args=run.args, echo=run.echo, trace_enabled=run.trace)

+	 	# Wait until initialized

+		p.expect(const.PROMPT)

+		p.send("echo 1")

+		p.send("echo 1")

+		# add running instance

+		script.test.process.append(p)

+		# run initial script

+		for cmd in run.cmds:

+			if len(cmd) >= 3 and cmd[2]!="":

+				print "====== " + cmd[2] + " ======"

+			if len(cmd) >= 1 and cmd[0]!="":

+				p.send(cmd[0])

+			if len(cmd) >= 2 and cmd[1]!="":

+				p.expect(cmd[1])

+

+	except TestError, e:

+		handle_error(e.desc, script.test)

+

+# Run the test function

+if script.test.test_func != None:

+	try:

+		script.test.test_func(script.test)

+	except TestError, e:

+		handle_error(e.desc, script.test)

+

+# Shutdown all instances

+time.sleep(2)

+for p in script.test.process:

+	p.send("q")

+	time.sleep(0.5)

+	p.expect(const.DESTROYED)

+	p.wait()

+

+# Done

+print "Test " + script.test.title + " completed successfully"

+sys.exit(0)

+

diff --git a/pjsip-apps/src/test-pjsua/scripts-call/100_simplecall.py b/pjsip-apps/src/test-pjsua/scripts-call/100_simplecall.py
new file mode 100644
index 00000000..52dcfed0
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/scripts-call/100_simplecall.py
@@ -0,0 +1,10 @@
+# $Id$

+#

+import inc_cfg

+

+# Simple call

+config = inc_cfg.CallConfig(

+		title = "Basic call",

+		callee_cfg = inc_cfg.Config(arg="--null-audio"),

+		caller_cfg = inc_cfg.Config(arg="--null-audio")

+		)

diff --git a/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_0_1.py b/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_0_1.py
new file mode 100644
index 00000000..319c4701
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_0_1.py
@@ -0,0 +1,10 @@
+# $Id$

+#

+import inc_cfg

+

+# Simple call

+config = inc_cfg.CallConfig(

+		title = "Callee=no SRTP, caller=optional SRTP",

+		callee_cfg = inc_cfg.Config(arg="--null-audio"),

+		caller_cfg = inc_cfg.Config(arg="--null-audio --use-srtp=1 --srtp-secure=0")

+		)

diff --git a/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_1_0.py b/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_1_0.py
new file mode 100644
index 00000000..ce886a69
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_1_0.py
@@ -0,0 +1,10 @@
+# $Id$

+#

+import inc_cfg

+

+# Simple call

+config = inc_cfg.CallConfig(

+		title = "Callee=optional SRTP, caller=no SRTP",

+		callee_cfg = inc_cfg.Config(arg="--null-audio --use-srtp=1 --srtp-secure=0"),

+		caller_cfg = inc_cfg.Config(arg="--null-audio")

+		)

diff --git a/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_1_1.py b/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_1_1.py
new file mode 100644
index 00000000..6a58b532
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/scripts-call/150_srtp_1_1.py
@@ -0,0 +1,10 @@
+# $Id$

+#

+import inc_cfg

+

+# Simple call

+config = inc_cfg.CallConfig(

+		title = "Callee=optional SRTP, caller=optional SRTP",

+		callee_cfg = inc_cfg.Config(arg="--null-audio --use-srtp=1 --srtp-secure=0"),

+		caller_cfg = inc_cfg.Config(arg="--null-audio --use-srtp=1 --srtp-secure=0")

+		)

diff --git a/pjsip-apps/src/test-pjsua/scripts-run/100_simple.py b/pjsip-apps/src/test-pjsua/scripts-run/100_simple.py
new file mode 100644
index 00000000..e3563039
--- /dev/null
+++ b/pjsip-apps/src/test-pjsua/scripts-run/100_simple.py
@@ -0,0 +1,8 @@
+# $Id$

+#

+# Just about the simple pjsua command line parameter, which should

+# never fail in any circumstances

+import inc_cfg

+

+config = inc_cfg.Config(arg = "--null-audio --local-port 0 --rtp-port 0")

+