Replace Berkeley DB with SQLite 3

There were some bugs in the very ancient version of Berkeley DB that Asterisk
used. Instead of spending the time tracking down the bugs in the Berkeley code
we move to the much better documented SQLite 3.

Conversion of the old astdb happens at runtime by running the included
astdb2sqlite3 utility. The ast_db API with SQLite 3 backend should behave
identically to the old Berkeley backend, but in the future we could offer a
much more robust interface.

We do not include the SQLite 3 library in the source tree, but instead rely
upon the distribution-provided libraries. SQLite is so ubiquitous that this
should not place undue burden on administrators.


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@326589 65c4cc65-6c06-0410-ace0-fbb531ad65f3

1 files changed, 377 insertions, 241 deletions

diff --git a/main/db.c b/main/db.c
index a0efc8c05..a1fff2146 100644
--- a/main/db.c
+++ b/main/db.c
@@ -20,11 +20,11 @@
  *
  * \brief ASTdb Management
  *
- * \author Mark Spencer <markster@digium.com> 
+ * \author Mark Spencer <markster@digium.com>
  *
  * \note DB3 is licensed under Sleepycat Public License and is thus incompatible
- * with GPL.  To avoid having to make another exception (and complicate 
- * licensing even further) we elect to use DB1 which is BSD licensed 
+ * with GPL.  To avoid having to make another exception (and complicate
+ * licensing even further) we elect to use DB1 which is BSD licensed
  */
 
 #include "asterisk.h"
@@ -34,8 +34,12 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 #include "asterisk/_private.h"
 #include "asterisk/paths.h"	/* use ast_config_AST_DB */
 #include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
 #include <signal.h>
 #include <dirent.h>
+#include <sqlite3.h>
 
 #include "asterisk/channel.h"
 #include "asterisk/file.h"
@@ -44,9 +48,7 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 #include "asterisk/astdb.h"
 #include "asterisk/cli.h"
 #include "asterisk/utils.h"
-#include "asterisk/lock.h"
 #include "asterisk/manager.h"
-#include "db1-ast/include/db.h"
 
 /*** DOCUMENTATION
 	<manager name="DBGet" language="en_US">
@@ -101,242 +103,362 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
  ***/
 
 #define MAX_DB_FIELD 256
-
-static DB *astdb;
 AST_MUTEX_DEFINE_STATIC(dblock);
 static ast_cond_t dbcond;
-typedef int (*process_keys_cb)(DBT *key, DBT *value, const char *filter, void *data);
+static sqlite3 *astdb;
+static pthread_t syncthread;
+static int doexit;
 
 static void db_sync(void);
 
-static int dbinit(void) 
+#define DEFINE_SQL_STATEMENT(stmt,sql) static sqlite3_stmt *stmt; \
+	const char stmt##_sql[] = sql;
+
+DEFINE_SQL_STATEMENT(put_stmt, "INSERT OR REPLACE INTO astdb (key, value) VALUES (?, ?)")
+DEFINE_SQL_STATEMENT(get_stmt, "SELECT value FROM astdb WHERE key=?")
+DEFINE_SQL_STATEMENT(del_stmt, "DELETE FROM astdb WHERE key=?")
+DEFINE_SQL_STATEMENT(deltree_stmt, "DELETE FROM astdb WHERE key LIKE ? || '/' || '%'")
+DEFINE_SQL_STATEMENT(deltree_all_stmt, "DELETE FROM astdb")
+DEFINE_SQL_STATEMENT(gettree_stmt, "SELECT key, value FROM astdb WHERE key LIKE ? || '/' || '%'")
+DEFINE_SQL_STATEMENT(gettree_all_stmt, "SELECT key, value FROM astdb")
+DEFINE_SQL_STATEMENT(showkey_stmt, "SELECT key, value FROM astdb WHERE key LIKE '%' || '/' || ?")
+DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key VARCHAR(256), value VARCHAR(256), PRIMARY KEY(key))")
+
+static int init_stmt(sqlite3_stmt **stmt, const char *sql, size_t len)
 {
-	if (!astdb && !(astdb = dbopen(ast_config_AST_DB, O_CREAT | O_RDWR, AST_FILE_MODE, DB_BTREE, NULL))) {
-		ast_log(LOG_WARNING, "Unable to open Asterisk database '%s': %s\n", ast_config_AST_DB, strerror(errno));
+	ast_mutex_lock(&dblock);
+	if (sqlite3_prepare_v2(astdb, sql, len, stmt, NULL) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't prepare statement '%s': %s\n", sql, sqlite3_errmsg(astdb));
+		ast_mutex_unlock(&dblock);
 		return -1;
 	}
+	ast_mutex_unlock(&dblock);
+
 	return 0;
 }
 
-
-static inline int keymatch(const char *key, const char *prefix)
+static int init_statements(void)
 {
-	int preflen = strlen(prefix);
-	if (!preflen)
-		return 1;
-	if (!strcasecmp(key, prefix))
-		return 1;
-	if ((strlen(key) > preflen) && !strncasecmp(key, prefix, preflen)) {
-		if (key[preflen] == '/')
-			return 1;
-	}
-	return 0;
+	/* Don't initialize create_astdb_statment here as the astdb table needs to exist
+	 * brefore these statments can be initialized */
+	return init_stmt(&get_stmt, get_stmt_sql, sizeof(get_stmt_sql))
+	|| init_stmt(&del_stmt, del_stmt_sql, sizeof(del_stmt_sql))
+	|| init_stmt(&deltree_stmt, deltree_stmt_sql, sizeof(deltree_stmt_sql))
+	|| init_stmt(&deltree_all_stmt, deltree_all_stmt_sql, sizeof(deltree_all_stmt_sql))
+	|| init_stmt(&gettree_stmt, gettree_stmt_sql, sizeof(gettree_stmt_sql))
+	|| init_stmt(&gettree_all_stmt, gettree_all_stmt_sql, sizeof(gettree_all_stmt_sql))
+	|| init_stmt(&showkey_stmt, showkey_stmt_sql, sizeof(showkey_stmt_sql))
+	|| init_stmt(&put_stmt, put_stmt_sql, sizeof(put_stmt_sql));
 }
 
-static inline int subkeymatch(const char *key, const char *suffix)
+static int convert_bdb_to_sqlite3(void)
 {
-	int suffixlen = strlen(suffix);
-	if (suffixlen) {
-		const char *subkey = key + strlen(key) - suffixlen;
-		if (subkey < key)
-			return 0;
-		if (!strcasecmp(subkey, suffix))
-			return 1;
-	}
-	return 0;
+	char *cmd;
+	int res;
+
+	ast_asprintf(&cmd, "astdb2sqlite3 '%s'\n", ast_config_AST_DB);
+	res = ast_safe_system(cmd);
+	ast_free(cmd);
+
+	return res;
 }
 
-static const char *dbt_data2str(DBT *dbt)
+static int db_create_astdb(void)
 {
-	char *data = "";
+	int res = 0;
 
-	if (dbt->size) {
-		data = dbt->data;
-		data[dbt->size - 1] = '\0';
+	if (!create_astdb_stmt) {
+		init_stmt(&create_astdb_stmt, create_astdb_stmt_sql, sizeof(create_astdb_stmt_sql));
 	}
 
-	return data;
-}
+	ast_mutex_lock(&dblock);
+	if (sqlite3_step(create_astdb_stmt) != SQLITE_DONE) {
+		ast_log(LOG_WARNING, "Couldn't create astdb table: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	}
+	sqlite3_reset(create_astdb_stmt);
+	db_sync();
+	ast_mutex_unlock(&dblock);
 
-static inline const char *dbt_data2str_full(DBT *dbt, const char *def)
-{
-	return S_OR(dbt_data2str(dbt), def);
+	return res;
 }
 
-static int process_db_keys(process_keys_cb cb, void *data, const char *filter, int sync)
+static int db_open(void)
 {
-	DBT key = { 0, }, value = { 0, }, last_key = { 0, };
-	int counter = 0;
-	int res, last = 0;
-	char last_key_s[MAX_DB_FIELD];
+	char *dbname;
+	struct stat dont_care;
+
+	if (!(dbname = alloca(strlen(ast_config_AST_DB) + sizeof(".sqlite3")))) {
+		return -1;
+	}
+	strcpy(dbname, ast_config_AST_DB);
+	strcat(dbname, ".sqlite3");
+
+	if (stat(dbname, &dont_care) && !stat(ast_config_AST_DB, &dont_care)) {
+		if (convert_bdb_to_sqlite3()) {
+			ast_log(LOG_ERROR, "*** Database conversion failed!\n");
+			ast_log(LOG_ERROR, "*** Asterisk now uses SQLite3 for its internal\n");
+			ast_log(LOG_ERROR, "*** database. Conversion from the old astdb\n");
+			ast_log(LOG_ERROR, "*** failed. Most likely the astdb2sqlite3 utility\n");
+			ast_log(LOG_ERROR, "*** was not selected for build. To convert the\n");
+			ast_log(LOG_ERROR, "*** old astdb, please delete '%s'\n", dbname);
+			ast_log(LOG_ERROR, "*** and re-run 'make menuselect' and select astdb2sqlite3\n");
+			ast_log(LOG_ERROR, "*** in the Utilities section, then 'make && make install'.\n");
+			sleep(5);
+		} else {
+			ast_log(LOG_NOTICE, "Database conversion succeeded!\n");
+		}
+	}
 
 	ast_mutex_lock(&dblock);
-	if (dbinit()) {
+	if (sqlite3_open_v2(dbname, &astdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Unable to open Asterisk database '%s': %s\n", dbname, sqlite3_errmsg(astdb));
+		sqlite3_close(astdb);
 		ast_mutex_unlock(&dblock);
 		return -1;
 	}
+	ast_mutex_unlock(&dblock);
 
-	/* Somehow, the database can become corrupted such that astdb->seq will continue looping through
-	 * the database indefinitely. The pointer to last_key.data ends up getting re-used by the BDB lib
-	 * so this specifically queries for the last entry, makes a copy of the key, and then uses it as
-	 * a sentinel to avoid repeatedly looping over the list. */
+	return 0;
+}
 
-	if (astdb->seq(astdb, &last_key, &value, R_LAST)) {
-		/* Empty database */
-		ast_mutex_unlock(&dblock);
+static int db_init(void)
+{
+	if (astdb) {
 		return 0;
 	}
 
-	memcpy(last_key_s, last_key.data, MIN(last_key.size - 1, sizeof(last_key_s)));
-	last_key_s[last_key.size - 1] = '\0';
-	for (res = astdb->seq(astdb, &key, &value, R_FIRST);
-			!res;
-			res = astdb->seq(astdb, &key, &value, R_NEXT)) {
-		/* The callback might delete the key, so we have to check it before calling */
-		last = !strcmp(dbt_data2str_full(&key, "<bad key>"), last_key_s);
-		counter += cb(&key, &value, filter, data);
-		if (last) {
-			break;
-		}
+	if (db_open() || db_create_astdb() || init_statements()) {
+		return -1;
 	}
 
-	if (sync) {
-		db_sync();
+	return 0;
+}
+
+/* We purposely don't lock around the sqlite3 call because the transaction
+ * calls will be called with the database lock held. For any other use, make
+ * sure to take the dblock yourself. */
+static int db_execute_sql(const char *sql, int (*callback)(void *, int, char **, char **), void *arg)
+{
+	char *errmsg = NULL;
+	int res =0;
+
+	sqlite3_exec(astdb, sql, callback, arg, &errmsg);
+	if (errmsg) {
+		ast_log(LOG_WARNING, "Error executing SQL: %s\n", errmsg);
+		sqlite3_free(errmsg);
+		res = -1;
 	}
 
-	ast_mutex_unlock(&dblock);
+	return res;
+}
 
-	return counter;
+static int ast_db_begin_transaction(void)
+{
+	return db_execute_sql("BEGIN TRANSACTION", NULL, NULL);
 }
 
-static int db_deltree_cb(DBT *key, DBT *value, const char *filter, void *data)
+static int ast_db_commit_transaction(void)
 {
-	int res = 0;
+	return db_execute_sql("COMMIT", NULL, NULL);
+}
 
-	if (keymatch(dbt_data2str_full(key, "<bad key>"), filter)) {
-		astdb->del(astdb, key, 0);
-		res = 1;
-	}
-	return res;
+static int ast_db_rollback_transaction(void)
+{
+	return db_execute_sql("ROLLBACK", NULL, NULL);
 }
 
-int ast_db_deltree(const char *family, const char *keytree)
+int ast_db_put(const char *family, const char *key, const char *value)
 {
-	char prefix[MAX_DB_FIELD];
+	char fullkey[MAX_DB_FIELD];
+	size_t fullkey_len;
+	int res = 0;
 
-	if (family) {
-		if (keytree) {
-			snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
-		} else {
-			snprintf(prefix, sizeof(prefix), "/%s", family);
-		}
-	} else if (keytree) {
+	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
+		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
 		return -1;
-	} else {
-		prefix[0] = '\0';
 	}
 
-	return process_db_keys(db_deltree_cb, NULL, prefix, 1);
+	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_bind_text(put_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_bind_text(put_stmt, 2, value, -1, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind value to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(put_stmt) != SQLITE_DONE) {
+		ast_log(LOG_WARNING, "Couldn't execute statment: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	}
+
+	sqlite3_reset(put_stmt);
+	db_sync();
+	ast_mutex_unlock(&dblock);
+
+	return res;
 }
 
-int ast_db_put(const char *family, const char *keys, const char *value)
+int ast_db_get(const char *family, const char *key, char *value, int valuelen)
 {
+	const unsigned char *result;
 	char fullkey[MAX_DB_FIELD];
-	DBT key, data;
-	int res, fullkeylen;
+	size_t fullkey_len;
+	int res = 0;
 
-	ast_mutex_lock(&dblock);
-	if (dbinit()) {
-		ast_mutex_unlock(&dblock);
+	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
+		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
 		return -1;
 	}
 
-	fullkeylen = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, keys);
-	memset(&key, 0, sizeof(key));
-	memset(&data, 0, sizeof(data));
-	key.data = fullkey;
-	key.size = fullkeylen + 1;
-	data.data = (char *) value;
-	data.size = strlen(value) + 1;
-	res = astdb->put(astdb, &key, &data, 0);
-	db_sync();
+	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_bind_text(get_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(get_stmt) != SQLITE_ROW) {
+		ast_debug(1, "Unable to find key '%s' in family '%s'\n", key, family);
+		res = -1;
+	} else if (!(result = sqlite3_column_text(get_stmt, 0))) {
+		ast_log(LOG_WARNING, "Couldn't get value\n");
+		res = -1;
+	} else {
+		strncpy(value, (const char *) result, valuelen);
+	}
+	sqlite3_reset(get_stmt);
 	ast_mutex_unlock(&dblock);
-	if (res)
-		ast_log(LOG_WARNING, "Unable to put value '%s' for key '%s' in family '%s'\n", value, keys, family);
 
 	return res;
 }
 
-int ast_db_get(const char *family, const char *keys, char *value, int valuelen)
+int ast_db_del(const char *family, const char *key)
 {
-	char fullkey[MAX_DB_FIELD] = "";
-	DBT key, data;
-	int res, fullkeylen;
+	char fullkey[MAX_DB_FIELD];
+	size_t fullkey_len;
+	int res = 0;
 
-	ast_mutex_lock(&dblock);
-	if (dbinit()) {
-		ast_mutex_unlock(&dblock);
+	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
+		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
 		return -1;
 	}
 
-	fullkeylen = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, keys);
-	memset(&key, 0, sizeof(key));
-	memset(&data, 0, sizeof(data));
-	memset(value, 0, valuelen);
-	key.data = fullkey;
-	key.size = fullkeylen + 1;
+	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
 
-	res = astdb->get(astdb, &key, &data, 0);
+	ast_mutex_lock(&dblock);
+	if (sqlite3_bind_text(del_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(del_stmt) != SQLITE_DONE) {
+		ast_debug(1, "Unable to find key '%s' in family '%s'\n", key, family);
+		res = -1;
+	}
+	sqlite3_reset(del_stmt);
+	db_sync();
+	ast_mutex_unlock(&dblock);
 
-	/* Be sure to NULL terminate our data either way */
-	if (res) {
-		ast_debug(1, "Unable to find key '%s' in family '%s'\n", keys, family);
-	} else {
-#if 0
-		printf("Got value of size %d\n", data.size);
-#endif
-		if (data.size) {
-			((char *)data.data)[data.size - 1] = '\0';
-			/* Make sure that we don't write too much to the dst pointer or we don't read too much from the source pointer */
-			ast_copy_string(value, data.data, (valuelen > data.size) ? data.size : valuelen);
+	return res;
+}
+
+int ast_db_deltree(const char *family, const char *subfamily)
+{
+	sqlite3_stmt *stmt = deltree_stmt;
+	char prefix[MAX_DB_FIELD];
+	int res = 0;
+
+	if (!ast_strlen_zero(family)) {
+		if (!ast_strlen_zero(subfamily)) {
+			/* Family and key tree */
+			snprintf(prefix, sizeof(prefix), "/%s/%s", family, subfamily);
 		} else {
-			ast_log(LOG_NOTICE, "Strange, empty value for /%s/%s\n", family, keys);
+			/* Family only */
+			snprintf(prefix, sizeof(prefix), "/%s", family);
 		}
+	} else {
+		prefix[0] = '\0';
+		stmt = deltree_all_stmt;
 	}
 
-	/* Data is not fully isolated for concurrency, so the lock must be extended
-	 * to after the copy to the output buffer. */
+	ast_mutex_lock(&dblock);
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(stmt) != SQLITE_DONE) {
+		ast_log(LOG_WARNING, "Couldn't execute stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	}
+	res = sqlite3_changes(astdb);
+	sqlite3_reset(stmt);
+	db_sync();
 	ast_mutex_unlock(&dblock);
 
 	return res;
 }
 
-int ast_db_del(const char *family, const char *keys)
+struct ast_db_entry *ast_db_gettree(const char *family, const char *subfamily)
 {
-	char fullkey[MAX_DB_FIELD];
-	DBT key;
-	int res, fullkeylen;
+	char prefix[MAX_DB_FIELD];
+	sqlite3_stmt *stmt = gettree_stmt;
+	struct ast_db_entry *cur, *last = NULL, *ret = NULL;
+
+	if (!ast_strlen_zero(family)) {
+		if (!ast_strlen_zero(subfamily)) {
+			/* Family and key tree */
+			snprintf(prefix, sizeof(prefix), "/%s/%s", family, subfamily);
+		} else {
+			/* Family only */
+			snprintf(prefix, sizeof(prefix), "/%s", family);
+		}
+	} else {
+		prefix[0] = '\0';
+		stmt = gettree_all_stmt;
+	}
 
 	ast_mutex_lock(&dblock);
-	if (dbinit()) {
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
+		sqlite3_reset(stmt);
 		ast_mutex_unlock(&dblock);
-		return -1;
+		return NULL;
 	}
-	
-	fullkeylen = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, keys);
-	memset(&key, 0, sizeof(key));
-	key.data = fullkey;
-	key.size = fullkeylen + 1;
-	
-	res = astdb->del(astdb, &key, 0);
-	db_sync();
-	
+
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		const char *key_s, *value_s;
+		if (!(key_s = (const char *) sqlite3_column_text(stmt, 0))) {
+			break;
+		}
+		if (!(value_s = (const char *) sqlite3_column_text(stmt, 1))) {
+			break;
+		}
+		if (!(cur = ast_malloc(sizeof(*cur) + strlen(key_s) + strlen(value_s) + 2))) {
+			break;
+		}
+		cur->next = NULL;
+		cur->key = cur->data + strlen(value_s) + 1;
+		strcpy(cur->data, value_s);
+		strcpy(cur->key, key_s);
+		if (last) {
+			last->next = cur;
+		} else {
+			ret = cur;
+		}
+		last = cur;
+	}
+	sqlite3_reset(stmt);
 	ast_mutex_unlock(&dblock);
 
-	if (res) {
-		ast_debug(1, "Unable to find key '%s' in family '%s'\n", keys, family);
+	return ret;
+}
+
+void ast_db_freetree(struct ast_db_entry *dbe)
+{
+	struct ast_db_entry *last;
+	while (dbe) {
+		last = dbe;
+		dbe = dbe->next;
+		ast_free(last);
 	}
-	return res;
 }
 
 static char *handle_cli_database_put(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
@@ -429,8 +551,8 @@ static char *handle_cli_database_deltree(struct ast_cli_entry *e, int cmd, struc
 	case CLI_INIT:
 		e->command = "database deltree";
 		e->usage =
-			"Usage: database deltree <family> [keytree]\n"
-			"       Deletes a family or specific keytree within a family\n"
+			"Usage: database deltree <family> [subfamily]\n"
+			"       Deletes a family or specific subfamily within a family\n"
 			"       in the Asterisk database.\n";
 		return NULL;
 	case CLI_GENERATE:
@@ -452,32 +574,19 @@ static char *handle_cli_database_deltree(struct ast_cli_entry *e, int cmd, struc
 	return CLI_SUCCESS;
 }
 
-static int db_show_cb(DBT *key, DBT *value, const char *filter, void *data)
-{
-	struct ast_cli_args *a = data;
-	const char *key_s = dbt_data2str_full(key, "<bad key>");
-	const char *value_s = dbt_data2str_full(value, "<bad value>");
-
-	if (keymatch(key_s, filter)) {
-		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
-		return 1;
-	}
-
-	return 0;
-}
-
 static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
 	char prefix[MAX_DB_FIELD];
 	int counter = 0;
+	sqlite3_stmt *stmt = gettree_stmt;
 
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "database show";
 		e->usage =
-			"Usage: database show [family [keytree]]\n"
+			"Usage: database show [family [subfamily]]\n"
 			"       Shows Asterisk database contents, optionally restricted\n"
-			"       to a given family, or family and keytree.\n";
+			"       to a given family, or family and subfamily.\n";
 		return NULL;
 	case CLI_GENERATE:
 		return NULL;
@@ -492,118 +601,123 @@ static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct a
 	} else if (a->argc == 2) {
 		/* Neither */
 		prefix[0] = '\0';
+		stmt = gettree_all_stmt;
+
 	} else {
 		return CLI_SHOWUSAGE;
 	}
 
-	if((counter = process_db_keys(db_show_cb, a, prefix, 0)) < 0) {
-		ast_cli(a->fd, "Database unavailable\n");
-		return CLI_SUCCESS;
+	ast_mutex_lock(&dblock);
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
+		sqlite3_reset(stmt);
+		ast_mutex_unlock(&dblock);
+		return NULL;
 	}
 
-	ast_cli(a->fd, "%d results found.\n", counter);
-	return CLI_SUCCESS;
-}
-
-static int db_showkey_cb(DBT *key, DBT *value, const char *filter, void *data)
-{
-	struct ast_cli_args *a = data;
-	const char *key_s = dbt_data2str_full(key, "<bad key>");
-	const char *value_s = dbt_data2str_full(value, "<bad value>");
-
-	if (subkeymatch(key_s, filter)) {
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		const char *key_s, *value_s;
+		if (!(key_s = (const char *) sqlite3_column_text(stmt, 0))) {
+			ast_log(LOG_WARNING, "Skipping invalid key!\n");
+			continue;
+		}
+		if (!(value_s = (const char *) sqlite3_column_text(stmt, 1))) {
+			ast_log(LOG_WARNING, "Skipping invalid value!\n");
+			continue;
+		}
+		++counter;
 		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
-		return 1;
 	}
 
-	return 0;
+	sqlite3_reset(stmt);
+	ast_mutex_unlock(&dblock);
+
+	ast_cli(a->fd, "%d results found.\n", counter);
+	return CLI_SUCCESS;
 }
 
 static char *handle_cli_database_showkey(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	char suffix[MAX_DB_FIELD];
 	int counter = 0;
 
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "database showkey";
 		e->usage =
-			"Usage: database showkey <keytree>\n"
+			"Usage: database showkey <subfamily>\n"
 			"       Shows Asterisk database contents, restricted to a given key.\n";
 		return NULL;
 	case CLI_GENERATE:
 		return NULL;
 	}
 
-	if (a->argc == 3) {
-		/* Key only */
-		snprintf(suffix, sizeof(suffix), "/%s", a->argv[2]);
-	} else {
+	if (a->argc != 3) {
 		return CLI_SHOWUSAGE;
 	}
 
-	if ((counter = process_db_keys(db_showkey_cb, a, suffix, 0)) < 0) {
-		ast_cli(a->fd, "Database unavailable\n");
-		return CLI_SUCCESS;
+	ast_mutex_lock(&dblock);
+	if (!ast_strlen_zero(a->argv[2]) && (sqlite3_bind_text(showkey_stmt, 1, a->argv[2], -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", a->argv[2], sqlite3_errmsg(astdb));
+		sqlite3_reset(showkey_stmt);
+		ast_mutex_unlock(&dblock);
+		return NULL;
+	}
+
+	while (sqlite3_step(showkey_stmt) == SQLITE_ROW) {
+		const char *key_s, *value_s;
+		if (!(key_s = (const char *) sqlite3_column_text(showkey_stmt, 0))) {
+			break;
+		}
+		if (!(value_s = (const char *) sqlite3_column_text(showkey_stmt, 1))) {
+			break;
+		}
+		++counter;
+		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
 	}
+	sqlite3_reset(showkey_stmt);
+	ast_mutex_unlock(&dblock);
 
 	ast_cli(a->fd, "%d results found.\n", counter);
 	return CLI_SUCCESS;
 }
 
-static int db_gettree_cb(DBT *key, DBT *value, const char *filter, void *data)
+static int display_results(void *arg, int columns, char **values, char **colnames)
 {
-	struct ast_db_entry **ret = data;
-	struct ast_db_entry *cur;
-	const char *key_s = dbt_data2str_full(key, "<bad key>");
-	const char *value_s = dbt_data2str_full(value, "<bad value>");
-	size_t key_slen = strlen(key_s) + 1, value_slen = strlen(value_s) + 1;
+	struct ast_cli_args *a = arg;
+	size_t x;
 
-	if (keymatch(key_s, filter) && (cur = ast_malloc(sizeof(*cur) + key_slen + value_slen))) {
-		cur->next = *ret;
-		cur->key = cur->data + value_slen;
-		strcpy(cur->data, value_s);
-		strcpy(cur->key, key_s);
-		*ret = cur;
-		return 1;
+	for (x = 0; x < columns; x++) {
+		ast_cli(a->fd, "%-5s: %-50s\n", colnames[x], values[x]);
 	}
+	ast_cli(a->fd, "\n");
 
 	return 0;
 }
 
-struct ast_db_entry *ast_db_gettree(const char *family, const char *keytree)
+static char *handle_cli_database_query(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	char prefix[MAX_DB_FIELD];
-	struct ast_db_entry *ret = NULL;
 
-	if (!ast_strlen_zero(family)) {
-		if (!ast_strlen_zero(keytree)) {
-			/* Family and key tree */
-			snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
-		} else {
-			/* Family only */
-			snprintf(prefix, sizeof(prefix), "/%s", family);
-		}
-	} else {
-		prefix[0] = '\0';
+	switch (cmd) {
+	case CLI_INIT:
+		e->command = "database query";
+		e->usage =
+			"Usage: database query \"<SQL Statement>\"\n"
+			"       Run a user-specified SQL query on the database. Be careful.\n";
+		return NULL;
+	case CLI_GENERATE:
+		return NULL;
 	}
 
-	if (process_db_keys(db_gettree_cb, &ret, prefix, 0) < 0) {
-		ast_log(LOG_WARNING, "Database unavailable\n");
-		return NULL;
+	if (a->argc != 3) {
+		return CLI_SHOWUSAGE;
 	}
 
-	return ret;
-}
+	ast_mutex_lock(&dblock);
+	db_execute_sql(a->argv[2], display_results, a);
+	db_sync(); /* Go ahead and sync the db in case they write */
+	ast_mutex_unlock(&dblock);
 
-void ast_db_freetree(struct ast_db_entry *dbe)
-{
-	struct ast_db_entry *last;
-	while (dbe) {
-		last = dbe;
-		dbe = dbe->next;
-		ast_free(last);
-	}
+	return CLI_SUCCESS;
 }
 
 static struct ast_cli_entry cli_database[] = {
@@ -612,7 +726,8 @@ static struct ast_cli_entry cli_database[] = {
 	AST_CLI_DEFINE(handle_cli_database_get,     "Gets database value"),
 	AST_CLI_DEFINE(handle_cli_database_put,     "Adds/updates database value"),
 	AST_CLI_DEFINE(handle_cli_database_del,     "Removes database key/value"),
-	AST_CLI_DEFINE(handle_cli_database_deltree, "Removes database keytree/values")
+	AST_CLI_DEFINE(handle_cli_database_deltree, "Removes database subfamily/values"),
+	AST_CLI_DEFINE(handle_cli_database_query,   "Run a user-specified query on the astdb"),
 };
 
 static int manager_dbput(struct mansession *s, const struct message *m)
@@ -726,7 +841,7 @@ static int manager_dbdeltree(struct mansession *s, const struct message *m)
 		astman_send_error(s, m, "Database entry not found");
 	else
 		astman_send_ack(s, m, "Key tree deleted successfully");
-	
+
 	return 0;
 }
 
@@ -754,27 +869,48 @@ static void db_sync(void)
 static void *db_sync_thread(void *data)
 {
 	ast_mutex_lock(&dblock);
+	ast_db_begin_transaction();
 	for (;;) {
+		/* We're ok with spurious wakeups, so we don't worry about a predicate */
 		ast_cond_wait(&dbcond, &dblock);
+		if (ast_db_commit_transaction()) {
+			ast_db_rollback_transaction();
+		}
+		if (doexit) {
+			ast_mutex_unlock(&dblock);
+			break;
+		}
+		ast_db_begin_transaction();
 		ast_mutex_unlock(&dblock);
 		sleep(1);
 		ast_mutex_lock(&dblock);
-		astdb->sync(astdb, 0);
 	}
 
 	return NULL;
 }
 
+static void astdb_atexit(void)
+{
+	doexit = 1;
+	db_sync();
+	pthread_join(syncthread, NULL);
+	ast_mutex_lock(&dblock);
+	sqlite3_close(astdb);
+	ast_mutex_unlock(&dblock);
+}
+
 int astdb_init(void)
 {
-	pthread_t dont_care;
+	if (db_init()) {
+		return -1;
+	}
 
 	ast_cond_init(&dbcond, NULL);
-	if (ast_pthread_create_background(&dont_care, NULL, db_sync_thread, NULL)) {
+	if (ast_pthread_create_background(&syncthread, NULL, db_sync_thread, NULL)) {
 		return -1;
 	}
 
-	dbinit();
+	ast_register_atexit(astdb_atexit);
 	ast_cli_register_multiple(cli_database, ARRAY_LEN(cli_database));
 	ast_manager_register_xml("DBGet", EVENT_FLAG_SYSTEM | EVENT_FLAG_REPORTING, manager_dbget);
 	ast_manager_register_xml("DBPut", EVENT_FLAG_SYSTEM, manager_dbput);