PSJIP - sip.conf to res_sip.conf script

Most, if not all, of the backing features of a conf file should now be
implemented (e.g. multi-line comments, includes, templates, etc...).  A
few of the options still need to be mapped.  Those are currently listed
in the 'sip_to_res_sip.py' file.

Things to do:

(1) There is more work to do here, at least for the sip.conf items that
aren't currently parsed. An issue will be created for that.
(2) All of the scripts should probably be passed through pylint and have
as many PEP8 issues fixed as possible.
(3) A public review is probably warranted at that point of the entire script.

Reported by: Matt Jordan


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@397567 65c4cc65-6c06-0410-ace0-fbb531ad65f3

3 files changed, 510 insertions, 278 deletions

diff --git a/contrib/scripts/sip_to_res_sip/astconfigparser.py b/contrib/scripts/sip_to_res_sip/astconfigparser.py
index f5baf13be..4a324e2ec 100644
--- a/contrib/scripts/sip_to_res_sip/astconfigparser.py
+++ b/contrib/scripts/sip_to_res_sip/astconfigparser.py
@@ -1,3 +1,6 @@
+import re
+
+from astdicts import OrderedDict
 from astdicts import MultiOrderedDict
 
 def merge_values(left, right, key):
@@ -21,93 +24,197 @@ class Section(MultiOrderedDict):
        added default sections. If not found at that point then a 'KeyError'
        exception is raised.
     """
-    def __init__(self, defaults = []):
+    count = 0
+
+    def __init__(self, defaults=None, templates=None):
         MultiOrderedDict.__init__(self)
-        self._defaults = defaults
+        # track an ordered id of sections
+        Section.count += 1
+        self.id = Section.count
+        self._defaults = [] if defaults is None else defaults
+        self._templates = [] if templates is None else templates
 
-    def __getitem__(self, key):
-        """Get the value for the given key. If it is not found in the 'self'
-           then check inside the defaults before declaring unable to locate."""
-        if key in self:
+    def __cmp__(self, other):
+        return cmp(self.id, other.id)
+
+    def get(self, key, from_self=True, from_templates=True, from_defaults=True):
+        if from_self and key in self:
             return MultiOrderedDict.__getitem__(self, key)
 
-        for default in self._defaults:
-            if key in default:
-                return default[key]
+        if from_templates:
+            if self in self._templates:
+                return []
+            for t in self._templates:
+                try:
+                    # fail if not found on the search - doing it this way
+                    # allows template's templates to be searched.
+                    return t.get(key, True, from_templates, from_defaults)
+                except KeyError:
+                    pass
+
+        if from_defaults:
+            for d in self._defaults:
+                try:
+                    return d.get(key, True, from_templates, from_defaults)
+                except KeyError:
+                    pass
 
         raise KeyError(key)
 
-    def keys(self):
+    def __getitem__(self, key):
+        """Get the value for the given key. If it is not found in the 'self'
+           then check inside templates and defaults before declaring raising
+           a KeyError exception.
+        """
+        return self.get(key)
+
+    def keys(self, self_only=False):
         res = MultiOrderedDict.keys(self)
+        if self_only:
+            return res
+
+        for d in self._templates:
+            for key in d.keys():
+                if key not in res:
+                    res.append(key)
+
         for d in self._defaults:
             for key in d.keys():
                 if key not in res:
                     res.append(key)
         return res
 
-    def add_default(self, default):
-        self._defaults.append(default)
+    def add_defaults(self, defaults):
+        defaults.sort()
+        for i in defaults:
+            self._defaults.insert(0, i)
+
+    def add_templates(self, templates):
+        templates.sort(reverse=True);
+        self._templates.extend(templates)
 
     def get_merged(self, key):
         """Return a list of values for a given key merged from default(s)"""
         # first merge key/values from defaults together
         merged = []
-        for i in self._defaults:
+        for i in reversed(self._defaults):
             if not merged:
                 merged = i
                 continue
             merged = merge_values(merged, i, key)
+
+        for i in reversed(self._templates):
+            if not merged:
+                merged = i
+                continue
+            merged = merge_values(merged, i, key)
+
         # then merge self in
         return merge_values(merged, self, key)
 
 ###############################################################################
 
-def remove_comment(line):
-    """Remove any commented elements from the given line"""
-    line = line.partition(COMMENT)[0]
-    return line.rstrip()
+COMMENT = ';'
+COMMENT_START = ';--'
+COMMENT_END = '--;'
+
+DEFAULTSECT = 'general'
+
+def remove_comment(line, is_comment):
+    """Remove any commented elements from the line."""
+    if not line: return line, is_comment
+
+    if is_comment:
+        part = line.partition(COMMENT_END)
+        if part[1]:
+            # found multi-line comment end check string after it
+            return remove_comment(part[2], False)
+        return "", True
+
+    part = line.partition(COMMENT_START)
+    if part[1]:
+        # found multi-line comment start check string before
+        # it to make sure there wasn't an eol comment in it
+        has_comment = part[0].partition(COMMENT)
+        if has_comment[1]:
+            # eol comment found return anything before it
+            return has_comment[0], False
+
+        # check string after it to see if the comment ends
+        line, is_comment = remove_comment(part[2], True)
+        if is_comment:
+            # return possible string data before comment
+            return part[0].strip(), True
+
+        # otherwise it was an embedded comment so combine
+        return ''.join([part[0].strip(), ' ', line]).rstrip(), False
+
+    # check for eol comment
+    return line.partition(COMMENT)[0].strip(), False
+
+def try_include(line):
+    """Checks to see if the given line is an include.  If so return the
+       included filename, otherwise None.
+    """
+    if not line.startswith('#'):
+        return None
+
+    # it is an include - get file name
+    try:
+        return line[line.index('"') + 1:line.rindex('"')]
+    except ValueError:
+        print "Invalid include - could not parse filename."
+        return None
 
 def try_section(line):
     """Checks to see if the given line is a section. If so return the section
        name, otherwise return 'None'.
     """
+    # leading spaces were stripped when checking for comments
     if not line.startswith('['):
-        return None
-
-    first, second, third = line.partition(']')
-    # TODO - third may contain template, parse to see if it is a template
-    #        or is a list of templates...return?
-    return first[1:]
+        return None, False, []
+
+    section, delim, templates = line.partition(']')
+    if not templates:
+        return section[1:], False, []
+
+    # strip out the parens and parse into an array
+    templates = templates.replace('(', "").replace(')', "").split(',')
+    # go ahead and remove extra whitespace
+    templates = [i.strip() for i in templates]
+    try:
+        templates.remove('!')
+        return section[1:], True, templates
+    except:
+        return section[1:], False, templates
 
 def try_option(line):
     """Parses the line as an option, returning the key/value pair."""
-    first, second, third = line.partition('=')
-    return first.strip(), third.strip()
+    data = re.split('=>?', line)
+    # should split in two (key/val), but either way use first two elements
+    return data[0].rstrip(), data[1].lstrip()
 
 ###############################################################################
 
-def get_value(mdict, key, index=-1):
-    """Given a multi-dict, retrieves a value for the given key. If the key only
-       holds a single value return that value. If the key holds more than one
-       value and an index is given that is greater than or equal to zero then
-       return the value at the index. Otherwise return the list of values."""
-    vals = mdict[key]
-    if len(vals) == 1:
-        return vals[0]
-    if index >= 0:
-        return vals[index]
-    return vals
-
-def find_value(mdicts, key, index=-1):
-    """Given a list of multi-dicts, try to find value(s) for the given key."""
-    if not isinstance(mdicts, list):
-        # given a single multi-dict
-        return get_value(mdicts, key, index)
-
-    for d in mdicts:
-        if key in d:
-            return d[key]
-    # not found, throw error
+def find_value(sections, key):
+    """Given a list of sections, try to find value(s) for the given key."""
+    # always start looking in the last one added
+    sections.sort(reverse=True);
+    for s in sections:
+        try:
+            # try to find in section and section's templates
+            return s.get(key, from_defaults=False)
+        except KeyError:
+            pass
+
+    # wasn't found in sections or a section's templates so check in defaults
+    for s in sections:
+        try:
+            # try to find in section's defaultsects
+            return s.get(key, from_self=False, from_templates=False)
+        except KeyError:
+            pass
+
     raise KeyError(key)
 
 def find_dict(mdicts, key, val):
@@ -115,80 +222,128 @@ def find_dict(mdicts, key, val):
        the given key/value pair."""
 
     def found(d):
-        # just check the first value of the key
-        return key in d and d[key][0] == val
-
-    if isinstance(mdicts, list):
-        try:
-            return [d for d in mdicts if found(d)][0]
-        except IndexError:
-            pass
-    elif found(mdicts):
-        return mdicts
+        return key in d and val in d[key]
+
+    try:
+        return [d for d in mdicts if found(d)][0]
+    except IndexError:
+        raise LookupError("Dictionary not located for key = %s, value = %s"
+                          % (key, val))
+
+def get_sections(parser, key, attr='_sections', searched=None):
+    if searched is None:
+        searched = []
+    if parser is None or parser in searched:
+        return []
+
+    try:
+        sections = getattr(parser, attr)
+        res = sections[key] if key in sections else []
+        searched.append(parser)
+        return res + get_sections(parser._includes, key, attr, searched) \
+            + get_sections(parser._parent, key, attr, searched)
+    except:
+        # assume ordereddict of parsers
+        res = []
+        for p in parser.itervalues():
+            res.extend(get_sections(p, key, attr, searched))
+        return res
 
-    raise LookupError("Dictionary not located for key = %s, value = %s"
-                      % (key, val))
+def get_defaults(parser, key):
+    return get_sections(parser, key, '_defaults')
+
+def write_dicts(file, mdicts):
+    for section, sect_list in mdicts.iteritems():
+        # every section contains a list of dictionaries
+        for sect in sect_list:
+            file.write("[%s]\n" % section)
+            for key, val_list in sect.iteritems():
+                # every value is also a list
+                for v in val_list:
+                    key_val = key
+                    if v is not None:
+                        key_val += " = " + str(v)
+                        file.write("%s\n" % (key_val))
+            file.write("\n")
 
 ###############################################################################
 
-COMMENT = ';'
-DEFAULTSECT = 'general'
-
 class MultiOrderedConfigParser:
-    def __init__(self):
-        self._default = MultiOrderedDict()
-        # sections contain dictionaries of dictionaries
+    def __init__(self, parent=None):
+        self._parent = parent
+        self._defaults = MultiOrderedDict()
         self._sections = MultiOrderedDict()
+        self._includes = OrderedDict()
 
-    def default(self):
-        return self._default
+    def defaults(self):
+        return self._defaults
+
+    def default(self, key):
+        """Retrieves a list of dictionaries for a default section."""
+        return get_defaults(self, key)
+
+    def add_default(self, key, template_keys=None):
+        """Adds a default section to defaults, returning the
+           default Section object.
+        """
+        if template_keys is None:
+            template_keys = []
+        return self.add_section(key, template_keys, self._defaults)
 
     def sections(self):
         return self._sections
 
-    def section(self, section, index=-1):
-        """Retrieves a section dictionary for the given section. If the section
-           holds only a single section dictionary return that dictionary. If
-           the section holds more than one dictionary and an index is given
-           that is greater than or equal to zero then return the dictionary at
-           the index. Otherwise return the list of dictionaries for the given
-           section name."""
-        try:
-            return get_value(self._sections, section, index)
-        except KeyError:
-            raise LookupError("section %r not found" % section)
-
-    def add_section(self, section, defaults=[]):
-        """Adds a section with the given name and defaults."""
-        self._sections[section] = res = Section(defaults)
+    def section(self, key):
+        """Retrieves a list of dictionaries for a section."""
+        return get_sections(self, key)
+
+    def add_section(self, key, template_keys=None, mdicts=None):
+        if template_keys is None:
+            template_keys = []
+        if mdicts is None:
+            mdicts = self._sections
+        res = Section()
+        for t in template_keys:
+            res.add_templates(get_defaults(self, t))
+        res.add_defaults(get_defaults(self, DEFAULTSECT))
+        mdicts.insert(0, key, res)
         return res
 
-    def get(self, key, section=DEFAULTSECT, index=-1):
-        """Retrieves a value for the given key from the given section. If the
-           key only holds a single value return that value. If the key holds
-           more than one value and an index is given that is greater than or
-           equal to zero then return the value at the index. Otherwise return
-           the list of values."""
+    def includes(self):
+        return self._includes
+
+    def add_include(self, filename, parser=None):
+        if filename in self._includes:
+            return self._includes[filename]
+
+        self._includes[filename] = res = \
+             MultiOrderedConfigParser(self) if parser is None else parser
+        return res;
+
+    def get(self, section, key):
+        """Retrieves the list of values from a section for a key."""
         try:
-            if section == DEFAULTSECT:
-                return get_value(self._default, key, index)
+            # search for the value in the list of sections
+            return find_value(self.section(section), key)
+        except KeyError:
+            pass
 
-            # search section(s)
-            return find_value(self.section(section), key, index)
+        try:
+            # section may be a default section so, search
+            # for the value in the list of defaults
+            return find_value(self.default(section), key)
         except KeyError:
-            # check default section if we haven't already
-            if section != DEFAULTSECT:
-                return self.get(key, DEFAULTSECT, index)
-            raise LookupError("key %r not found in section %r"
+            raise LookupError("key %r not found for section %r"
                               % (key, section))
 
-    def set(self, key, val, section=DEFAULTSECT):
+    def set(self, section, key, val):
         """Sets an option in the given section."""
-        if section == DEFAULTSECT:
-            self._default[key] = val
+        # TODO - set in multiple sections? (for now set in first)
+        # TODO - set in both sections and defaults?
+        if section in self._sections:
+            self.section(section)[0][key] = val
         else:
-            # for now only set value in first section
-            self.section(section, 0)[key] = val
+            self.defaults(section)[0][key] = val
 
     def read(self, filename):
         try:
@@ -198,45 +353,42 @@ class MultiOrderedConfigParser:
             print "Could not open file ", filename, " for reading"
 
     def _read(self, file, filename):
+        is_comment = False # used for multi-lined comments
         for line in file:
-            line = remove_comment(line)
+            line, is_comment = remove_comment(line, is_comment)
             if not line:
                 # line was empty or was a comment
                 continue
 
-            section = try_section(line)
+            include_name = try_include(line)
+            if include_name:
+                parser = self.add_include(include_name)
+                parser.read(include_name)
+                continue
+
+            section, is_template, templates = try_section(line)
             if section:
-                if section == DEFAULTSECT:
-                    sect = self._default
+                if section == DEFAULTSECT or is_template:
+                    sect = self.add_default(section, templates)
                 else:
-                    self._sections[section] = sect = Section([self._default])
-                    # TODO - if section has templates add those
-                    #        with sect.add_default
+                    sect = self.add_section(section, templates)
                 continue
 
             key, val = try_option(line)
             sect[key] = val
 
-    def write(self, filename):
+    def write(self, f):
         try:
-            with open(filename, 'wt') as file:
-                self._write(file)
-        except IOError:
-            print "Could not open file ", filename, " for writing"
-        pass
-
-    def _write(self, file):
-        # TODO - need to write out default section, but right now in
-        #        our case res_sip.conf has not default/general section
-        for section, sect_list in self._sections.iteritems():
-            # every section contains a list of dictionaries
-            for sect in sect_list:
-                file.write("[%s]\n" % section)
-                for key, val_list in sect.iteritems():
-                    # every value is also a list
-                    for v in val_list:
-                        key_val = key
-                        if (v is not None):
-                            key_val += " = " + str(v)
-                        file.write("%s\n" % (key_val))
-                file.write("\n")
+            for key, val in self._includes.iteritems():
+                val.write(key)
+                f.write('#include "%s"\n' % key)
+
+            f.write('\n')
+            write_dicts(f, self._defaults)
+            write_dicts(f, self._sections)
+        except:
+            try:
+                with open(f, 'wt') as fp:
+                    self.write(fp)
+            except IOError:
+                print "Could not open file ", f, " for writing"
diff --git a/contrib/scripts/sip_to_res_sip/astdicts.py b/contrib/scripts/sip_to_res_sip/astdicts.py
index 2a43c1163..ae630755d 100644
--- a/contrib/scripts/sip_to_res_sip/astdicts.py
+++ b/contrib/scripts/sip_to_res_sip/astdicts.py
@@ -265,11 +265,28 @@ class MultiOrderedDict(OrderedDict):
     def __init__(self, *args, **kwds):
         OrderedDict.__init__(self, *args, **kwds)
 
-    def __setitem__(self, key, val):
+    def __setitem__(self, key, val, i=None):
         if key not in self:
-            OrderedDict.__setitem__(self, key, [val])
-        elif val not in self[key]:
-            self[key].append(val)
+#            print "__setitem__ key = ", key, " val = ", val
+            OrderedDict.__setitem__(
+                self, key, val if isinstance(val, list) else [val])
+            return
+#        print "inserting key = ", key, " val = ", val
+        vals = self[key]
+        if i is None:
+            i = len(vals)
+
+        if not isinstance(val, list):
+            if val not in vals:
+                vals.insert(i, val)
+        else:
+            for j in val.reverse():
+                if j not in vals:
+                    vals.insert(i, j)
+
+
+    def insert(self, i, key, val):
+        self.__setitem__(key, val, i)
 
     def copy(self):
         # TODO - find out why for some reason copies
@@ -279,28 +296,3 @@ class MultiOrderedDict(OrderedDict):
             for v in val:
                 c[key] = v
         return c
-
-    # def update(self, other=None, **kwds):
-    #     if other is None:
-    #         pass
-
-    #     if isinstance(other, list):
-    #         for val in other:
-    #             update(self, val)
-    #         return
-
-    #     for key, val in other.iteritems():
-    #         # key = [ v1, v2, ...n ]
-    #         if key in self and len(self[key]) > 1:
-    #             # merge values adding only those not already in list
-    #             val = self[key] + [v for v in val if v not in self[key]]
-    #         OrderedDict.__setitem__(self, key, val)
-    #     # if hasattr(other, 'keys'):
-    #     #         other = other.keys()
-    #     # for (key, val) in obj.iteritems():
-    #     #     if key in self and len(self[key]) > 1:
-    #     #         # add only values not already in list
-    #     #         val = self[key] + [v for v in val if v not in self[key]]
-    #     #     OrderedDict.__setitem__(self, key, val)
-    #     if kwds:
-    #         self.update(kwds)
diff --git a/contrib/scripts/sip_to_res_sip/sip_to_res_sip.py b/contrib/scripts/sip_to_res_sip/sip_to_res_sip.py
index 13444a95c..ca253bdd0 100755
--- a/contrib/scripts/sip_to_res_sip/sip_to_res_sip.py
+++ b/contrib/scripts/sip_to_res_sip/sip_to_res_sip.py
@@ -1,17 +1,24 @@
 #!/usr/bin/python
 
-import astconfigparser
+###############################################################################
+# TODO:
+# (1) There is more work to do here, at least for the sip.conf items that
+#     aren't currently parsed. An issue will be created for that.
+# (2) All of the scripts should probably be passed through pylint and have
+#     as many PEP8 issues fixed as possible
+# (3) A public review is probably warranted at that point of the entire script
+###############################################################################
 
-# configuration parser for sip.conf
-sip = astconfigparser.MultiOrderedConfigParser()
+import optparse
+import astdicts
+import astconfigparser
 
-# configuration writer for res_sip.conf
-res_sip = astconfigparser.MultiOrderedConfigParser()
+PREFIX = 'res_sip_'
 
 ###############################################################################
 ### some utility functions
 ###############################################################################
-def section_by_type(section, type='endpoint'):
+def section_by_type(section, res_sip, type):
     """Finds a section based upon the given type, adding it if not found."""
     try:
         return astconfigparser.find_dict(
@@ -22,10 +29,11 @@ def section_by_type(section, type='endpoint'):
         sect['type'] = type
         return sect
 
-def set_value(key=None, val=None, section=None, type='endpoint'):
+def set_value(key=None, val=None, section=None, res_sip=None,
+              nmapped=None, type='endpoint'):
     """Sets the key to the value within the section in res_sip.conf"""
-    def _set_value(k, v, s):
-        set_value(key if key else k, v, s, type)
+    def _set_value(k, v, s, r, n):
+        set_value(key if key else k, v, s, r, n, type)
 
     # if no value or section return the set_value
     # function with the enclosed key and type
@@ -33,135 +41,151 @@ def set_value(key=None, val=None, section=None, type='endpoint'):
         return _set_value
 
     # otherwise try to set the value
-    section_by_type(section, type)[key] = val
+    section_by_type(section, res_sip, type)[key] = \
+        val[0] if isinstance(val, list) else val
 
-def merge_value(key=None, val=None, section=None,
-                type='endpoint', section_to=None):
+def merge_value(key=None, val=None, section=None, res_sip=None,
+                nmapped=None, type='endpoint', section_to=None):
     """Merge values from the given section with those from the default."""
-    def _merge_value(k, v, s):
-        merge_value(key if key else k, v, s, type, section_to)
+    def _merge_value(k, v, s, r, n):
+        merge_value(key if key else k, v, s, r, n, type, section_to)
 
     # if no value or section return the merge_value
     # function with the enclosed key and type
     if not val and not section:
         return _merge_value
 
-    # should return single section
-    sect = sip.section(section)
+    # should return a single value section list
+    sect = sip.section(section)[0]
     # for each merged value add it to res_sip.conf
     for i in sect.get_merged(key):
-        set_value(key, i, section_to if section_to else section, type)
+        set_value(key, i, section_to if section_to else section,
+                  res_sip, nmapped, type)
 
 def is_in(s, sub):
     """Returns true if 'sub' is in 's'"""
     return s.find(sub) != -1
 
+def non_mapped(nmapped):
+    def _non_mapped(section, key, val):
+        """Writes a non-mapped value from sip.conf to the non-mapped object."""
+        if section not in nmapped:
+            nmapped[section] = astconfigparser.Section()
+            if isinstance(val, list):
+                for v in val:
+                    # since coming from sip.conf we can assume
+                    # single section lists
+                    nmapped[section][0][key] = v
+            else:
+                nmapped[section][0][key] = val
+    return _non_mapped
+
 ###############################################################################
 ### mapping functions -
 ###      define f(key, val, section) where key/val are the key/value pair to
 ###      write to given section in res_sip.conf
 ###############################################################################
 
-def set_dtmfmode(key, val, section):
+def set_dtmfmode(key, val, section, res_sip, nmapped):
     """Sets the dtmfmode value.  If value matches allowable option in res_sip
        then map it, otherwise set it to none.
     """
     # available res_sip.conf values: frc4733, inband, info, none
     if val != 'inband' or val != 'info':
-        print "sip.conf: dtmfmode = %s did not fully map into " \
-              "res_sip.conf - setting to 'none'" % val
+        nmapped(section, key, val + " ; did not fully map - set to none")
         val = 'none'
-    set_value(key, val, section)
+    set_value(key, val, section, res_sip, nmapped)
 
-def from_nat(key, val, section):
+def from_nat(key, val, section, res_sip, nmapped):
     """Sets values from nat into the appropriate res_sip.conf options."""
     # nat from sip.conf can be comma separated list of values:
     # yes/no, [auto_]force_rport, [auto_]comedia
     if is_in(val, 'yes'):
-        set_value('rtp_symmetric', 'yes', section)
-        set_value('rewrite_contact', 'yes', section)
+        set_value('rtp_symmetric', 'yes', section, res_sip, nmapped)
+        set_value('rewrite_contact', 'yes', section, res_sip, nmapped)
     if is_in(val, 'comedia'):
-        set_value('rtp_symmetric', 'yes', section)
+        set_value('rtp_symmetric', 'yes', section, res_sip, nmapped)
     if is_in(val, 'force_rport'):
-        set_value('force_rport', 'yes', section)
-        set_value('rewrite_contact', 'yes', section)
+        set_value('force_rport', 'yes', section, res_sip, nmapped)
+        set_value('rewrite_contact', 'yes', section, res_sip, nmapped)
 
-def set_timers(key, val, section):
+def set_timers(key, val, section, res_sip, nmapped):
     """Sets the timers in res_sip.conf from the session-timers option
        found in sip.conf.
     """
     # res_sip.conf values can be yes/no, required, always
     if val == 'originate':
-        set_value('timers', 'always', section)
+        set_value('timers', 'always', section, res_sip, nmapped)
     elif val == 'accept':
-        set_value('timers', 'required', section)
+        set_value('timers', 'required', section, res_sip, nmapped)
     elif val == 'never':
-        set_value('timers', 'no', section)
+        set_value('timers', 'no', section, res_sip, nmapped)
     else:
-        set_value('timers', 'yes', section)
+        set_value('timers', 'yes', section, res_sip, nmapped)
 
-def set_direct_media(key, val, section):
+def set_direct_media(key, val, section, res_sip, nmapped):
     """Maps values from the sip.conf comma separated direct_media option
        into res_sip.conf direct_media options.
     """
     if is_in(val, 'yes'):
-        set_value('direct_media', 'yes', section)
+        set_value('direct_media', 'yes', section, res_sip, nmapped)
     if is_in(val, 'update'):
-        set_value('direct_media_method', 'update', section)
+        set_value('direct_media_method', 'update', section, res_sip, nmapped)
     if is_in(val, 'outgoing'):
-        set_value('directed_media_glare_mitigation', 'outgoing', section)
+        set_value('directed_media_glare_mitigation', 'outgoing', section, res_sip, nmapped)
     if is_in(val, 'nonat'):
-        set_value('disable_directed_media_on_nat', 'yes', section)
+        set_value('disable_directed_media_on_nat','yes', section, res_sip, nmapped)
+    if (val == 'no'):
+        set_value('direct_media', 'no', section, res_sip, nmapped)
 
-def from_sendrpid(key, val, section):
+def from_sendrpid(key, val, section, res_sip, nmapped):
     """Sets the send_rpid/pai values in res_sip.conf."""
     if val == 'yes' or val == 'rpid':
-        set_value('send_rpid', 'yes', section)
+        set_value('send_rpid', 'yes', section, res_sip, nmapped)
     elif val == 'pai':
-        set_value('send_pai', 'yes', section)
+        set_value('send_pai', 'yes', section, res_sip, nmapped)
 
-def set_media_encryption(key, val, section):
+def set_media_encryption(key, val, section, res_sip, nmapped):
     """Sets the media_encryption value in res_sip.conf"""
     if val == 'yes':
-        set_value('media_encryption', 'sdes', section)
+        set_value('media_encryption', 'sdes', section, res_sip, nmapped)
 
-def from_recordfeature(key, val, section):
+def from_recordfeature(key, val, section, res_sip, nmapped):
     """If record on/off feature is set to automixmon then set
        one_touch_recording, otherwise it can't be mapped.
     """
     if val == 'automixmon':
-        set_value('one_touch_recording', 'yes', section)
+        set_value('one_touch_recording', 'yes', section, res_sip, nmapped)
     else:
-        print "sip.conf: %s = %s could not be fully map " \
-              "one_touch_recording not set in res_sip.conf" % (key, val)
+        nmapped(section, key, val + " ; could not be fully mapped")
 
-def from_progressinband(key, val, section):
+def from_progressinband(key, val, section, res_sip, nmapped):
     """Sets the inband_progress value in res_sip.conf"""
     # progressinband can = yes/no/never
     if val == 'never':
         val = 'no'
-    set_value('inband_progress', val, section)
+    set_value('inband_progress', val, section, res_sip, nmapped)
 
-def from_host(key, val, section):
+def from_host(key, val, section, res_sip, nmapped):
     """Sets contact info in an AOR section in in res_sip.conf using 'host'
        data from sip.conf
     """
     # all aors have the same name as the endpoint so makes
     # it easy to endpoint's 'aors' value
-    set_value('aors', section, section)
+    set_value('aors', section, section, res_sip, nmapped)
     if val != 'dynamic':
-        set_value('contact', val, section, 'aor')
+        set_value('contact', val, section, res_sip, nmapped, 'aor')
     else:
-        set_value('max_contacts', 1, section, 'aor')
+        set_value('max_contacts', 1, section, res_sip, nmapped, 'aor')
 
-def from_subscribemwi(key, val, section):
+def from_subscribemwi(key, val, section, res_sip, nmapped):
     """Checks the subscribemwi value in sip.conf.  If yes places the
        mailbox value in mailboxes within the endpoint, otherwise puts
        it in the aor.
     """
-    mailboxes = sip.get('mailbox', section)
+    mailboxes = sip.get('mailbox', section, res_sip)
     type = 'endpoint' if val == 'yes' else 'aor'
-    set_value('mailboxes', mailboxes, section, type)
+    set_value('mailboxes', mailboxes, section, res_sip, nmapped, type)
 
 ###############################################################################
 
@@ -170,51 +194,51 @@ def from_subscribemwi(key, val, section):
 # connected_line_method
 
 # known sip.conf peer keys that can be mapped to a res_sip.conf section/key
-peer_map = {
+peer_map = [
     # sip.conf option      mapping function     res_sip.conf option(s)
     ###########################################################################
-    'context':            set_value,
-    'dtmfmode':           set_dtmfmode,
-    'disallow':           merge_value,
-    'allow':              merge_value,
-    'nat':                from_nat,            # rtp_symmetric, force_rport,
-                                               # rewrite_contact
-    'icesupport':         set_value('ice_support'),
-    'autoframing':        set_value('use_ptime'),
-    'outboundproxy':      set_value('outbound_proxy'),
-    'mohsuggest':         set_value,
-    'session-timers':     set_timers,          # timers
-    'session-minse':      set_value('timers_min_se'),
-    'session-expires':    set_value('timers_sess_expires'),
-    'externip':           set_value('external_media_address'),
-    'externhost':         set_value('external_media_address'),
+    ['context',            set_value],
+    ['dtmfmode',           set_dtmfmode],
+    ['disallow',           merge_value],
+    ['allow',              merge_value],
+    ['nat',                from_nat],            # rtp_symmetric, force_rport,
+                                                 # rewrite_contact
+    ['icesupport',         set_value('ice_support')],
+    ['autoframing',        set_value('use_ptime')],
+    ['outboundproxy',      set_value('outbound_proxy')],
+    ['mohsuggest',         set_value],
+    ['session-timers',     set_timers],          # timers
+    ['session-minse',      set_value('timers_min_se')],
+    ['session-expires',    set_value('timers_sess_expires')],
+    ['externip',           set_value('external_media_address')],
+    ['externhost',         set_value('external_media_address')],
     # identify_by ?
-    'direct_media':       set_direct_media,    # direct_media
-                                               # direct_media_method
-                                               # directed_media_glare_mitigation
-                                               # disable_directed_media_on_nat
-    'callerid':           set_value,           # callerid
-    'callingpres':        set_value('callerid_privacy'),
-    'cid_tag':            set_value('callerid_tag'),
-    'trustpid':           set_value('trust_id_inbound'),
-    'sendrpid':           from_sendrpid,       # send_pai, send_rpid
-    'send_diversion':     set_value,
-    'encrpytion':         set_media_encryption,
-    'use_avpf':           set_value,
-    'recordonfeature':    from_recordfeature,  # automixon
-    'recordofffeature':   from_recordfeature,  # automixon
-    'progressinband':     from_progressinband, # in_band_progress
-    'callgroup':          set_value,
-    'pickupgroup':        set_value,
-    'namedcallgroup':     set_value,
-    'namedpickupgroup':   set_value,
-    'busylevel':          set_value('devicestate_busy_at'),
+    ['directmedia',        set_direct_media],    # direct_media
+                                                 # direct_media_method
+                                                 # directed_media_glare_mitigation
+                                                 # disable_directed_media_on_nat
+    ['callerid',           set_value],           # callerid
+    ['callingpres',        set_value('callerid_privacy')],
+    ['cid_tag',            set_value('callerid_tag')],
+    ['trustpid',           set_value('trust_id_inbound')],
+    ['sendrpid',           from_sendrpid],       # send_pai, send_rpid
+    ['send_diversion',     set_value],
+    ['encrpytion',         set_media_encryption],
+    ['use_avpf',           set_value],
+    ['recordonfeature',    from_recordfeature],  # automixon
+    ['recordofffeature',   from_recordfeature],  # automixon
+    ['progressinband',     from_progressinband], # in_band_progress
+    ['callgroup',          set_value],
+    ['pickupgroup',        set_value],
+    ['namedcallgroup',     set_value],
+    ['namedpickupgroup',   set_value],
+    ['busylevel',          set_value('devicestate_busy_at')],
 
 ############################ maps to an aor ###################################
 
-    'host':               from_host,           # contact, max_contacts
-    'subscribemwi':       from_subscribemwi,   # mailboxes
-    'qualifyfreq':        set_value('qualify_frequency', type='aor'),
+    ['host',               from_host],           # contact, max_contacts
+    ['subscribemwi',       from_subscribemwi],   # mailboxes
+    ['qualifyfreq',        set_value('qualify_frequency', type='aor')],
 
 ############################# maps to auth#####################################
 #        type = auth
@@ -226,12 +250,12 @@ peer_map = {
 #        auth_type
 ######################### maps to acl/security ################################
 
-    'permit':             merge_value(type='security', section_to='acl'),
-    'deny':               merge_value(type='security', section_to='acl'),
-    'acl':                merge_value(type='security', section_to='acl'),
-    'contactpermit':      merge_value(type='security', section_to='acl'),
-    'contactdeny':        merge_value(type='security', section_to='acl'),
-    'contactacl':         merge_value(type='security', section_to='acl'),
+    ['permit',             merge_value(type='security', section_to='acl')],
+    ['deny',               merge_value(type='security', section_to='acl')],
+    ['acl',                merge_value(type='security', section_to='acl')],
+    ['contactpermit',      merge_value(type='security', section_to='acl')],
+    ['contactdeny',        merge_value(type='security', section_to='acl')],
+    ['contactacl',         merge_value(type='security', section_to='acl')],
 
 ########################### maps to transport #################################
 #        type = transport
@@ -271,34 +295,98 @@ peer_map = {
 #        type = identify
 #        endpoint
 #        match
-}
+]
 
-def map_peer(section):
-    for key, fun in peer_map.iteritems():
+def map_peer(sip, section, res_sip, nmapped):
+    for i in peer_map:
         try:
-            fun(key, sip.get(key, section), section)
+            # coming from sip.conf the values should mostly be a list with a
+            # single value.  In the few cases that they are not a specialized
+            # function (see merge_value) is used to retrieve the values.
+            i[1](i[0], sip.get(section, i[0])[0], section, res_sip, nmapped)
         except LookupError:
-            pass
-#            print "%s not found for section %s - putting nothing in res_sip.conf" % (key, section)
+            pass # key not found in sip.conf
 
-    # since we are pulling from sip.conf this should always return
-    # a single peer value and never a list of peers
-    peer = sip.section(section)
-    # loop through the peer and print out any that can't be mapped
-    # for key in peer.keys():
-    #     if key not in peer_map:
-    #         print "Peer: [{}] {} could not be mapped".format(section, key)
+def find_non_mapped(sections, nmapped):
+    for section, sect in sections.iteritems():
+        try:
+            # since we are pulling from sip.conf this should always
+            # be a single value list
+            sect = sect[0]
+            # loop through the section and store any values that were not mapped
+            for key in sect.keys(True):
+                for i in peer_map:
+                    if i[0] == key:
+                        break;
+                else:
+                    nmapped(section, key, sect[key])
+        except LookupError:
+            pass
 
-def convert():
+def convert(sip, filename, non_mappings):
+    res_sip = astconfigparser.MultiOrderedConfigParser()
+    non_mappings[filename] = astdicts.MultiOrderedDict()
+    nmapped = non_mapped(non_mappings[filename])
     for section in sip.sections():
         if section == 'authentication':
             pass
-        elif section != 'general':
-            map_peer(section)
+        else:
+            map_peer(sip, section, res_sip, nmapped)
+
+    find_non_mapped(sip.defaults(), nmapped)
+    find_non_mapped(sip.sections(), nmapped)
+
+    for key, val in sip.includes().iteritems():
+        res_sip.add_include(PREFIX + key, convert(val, PREFIX + key, non_mappings)[0])
+    return res_sip, non_mappings
+
+def write_res_sip(filename, res_sip, non_mappings):
+    try:
+        with open(filename, 'wt') as fp:
+            fp.write(';--\n')
+            fp.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n')
+            fp.write('Non mapped elements start\n')
+            fp.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n\n')
+            astconfigparser.write_dicts(fp, non_mappings[filename])
+            fp.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n')
+            fp.write('Non mapped elements end\n')
+            fp.write(';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n')
+            fp.write('--;\n\n')
+            # write out include file(s)
+            for key, val in res_sip.includes().iteritems():
+                write_res_sip(key, val, non_mappings)
+                fp.write('#include "%s"\n' % key)
+            fp.write('\n')
+            # write out mapped data elements
+            astconfigparser.write_dicts(fp, res_sip.defaults())
+            astconfigparser.write_dicts(fp, res_sip.sections())
+
+    except IOError:
+        print "Could not open file ", filename, " for writing"
 
 ###############################################################################
 
+def cli_options():
+    global PREFIX
+    usage = "usage: %prog [options] [input-file [output-file]]\n\n" \
+        "input-file defaults to 'sip.conf'\n" \
+        "output-file defaults to 'res_sip.conf'"
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option('-p', '--prefix', dest='prefix', default=PREFIX,
+                      help='output prefix for include files')
+
+    options, args = parser.parse_args()
+    PREFIX = options.prefix
+
+    sip_filename = args[0] if len(args) else 'sip.conf'
+    res_sip_filename = args[1] if len(args) == 2 else 'res_sip.conf'
+
+    return sip_filename, res_sip_filename
+
 if __name__ == "__main__":
-    sip.read('sip.conf')
-    convert()
-    res_sip.write('res_sip.conf')
+    sip_filename, res_sip_filename = cli_options()
+    # configuration parser for sip.conf
+    sip = astconfigparser.MultiOrderedConfigParser()
+    sip.read(sip_filename)
+    res_sip, non_mappings = convert(sip, res_sip_filename, dict())
+    write_res_sip(res_sip_filename, res_sip, non_mappings)