media cache: Add a core API and facade for a backend agnostic media cache

This patch adds a new API to the Asterisk core that acts as a media
cache. The core API itself is mostly a thin wrapper around some bucket
API provided implementation that itself acts as the mechanism of
retrieval for media. The media cache API in the core provides the
following:
 * A very thin in-memory cache of the active bucket_file items. Unlike a
   more traditional cache, it provides no expiration mechanisms. Most
   queries that hit the in-memory cache will also call into the bucket
   implementations as well. The bucket implementations are responsible
   for determining whether or not the active record is active and valid.
   This makes sense for the most likely implementation of a media cache
   backend, i.e., HTTP. The HTTP layer itself is the actual arbiter of
   whether or not a record is truly active; as such, the in-memory cache
   in the core has to defer to it.
 * The ability to create new items in the media cache from local
   resources. This allows for re-creation of items in the cache on
   restart.
 * Synchronization of items in the media cache to the AstDB. This
   also includes various pieces of important metadata.

The API provides sufficient access that higher level APIs, such as the
file or app APIs, do not have to worry about the semantics of the bucket
APIs when needing to playback a resource.

In addition, this patch provides unit tests for the media cache API. The
unit tests use a fake bucket backend to verify correctness.

Change-Id: I11227abbf14d8929eeb140ddd101dd5c3820391e

1 files changed, 490 insertions, 0 deletions

diff --git a/main/media_cache.c b/main/media_cache.c
new file mode 100644
index 000000000..47173c803
--- /dev/null
+++ b/main/media_cache.c
@@ -0,0 +1,490 @@
+/*
+ * Asterisk -- An open source telephony toolkit.
+ *
+ * Copyright (C) 2015, Matt Jordan
+ *
+ * Matt Jordan <mjordan@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+/*!
+ * \file
+ * \brief An in-memory media cache
+ *
+ * \author \verbatim Matt Jordan <mjordan@digium.com> \endverbatim
+ *
+ */
+
+/*** MODULEINFO
+	<support_level>core</support_level>
+ ***/
+
+#include "asterisk.h"
+
+ASTERISK_REGISTER_FILE()
+
+#include <sys/stat.h>
+#include "asterisk/config.h"
+#include "asterisk/bucket.h"
+#include "asterisk/astdb.h"
+#include "asterisk/media_cache.h"
+
+/*! The name of the AstDB family holding items in the cache. */
+#define AST_DB_FAMILY "MediaCache"
+
+/*! Length of 'MediaCache' + 2 '/' characters */
+#define AST_DB_FAMILY_LEN 12
+
+/*! Number of buckets in the ao2 container holding our media items */
+#define AO2_BUCKETS 61
+
+/*! Our one and only container holding media items */
+static struct ao2_container *media_cache;
+
+/*!
+ * \internal
+ * \brief Hashing function for file metadata
+ */
+static int media_cache_hash(const void *obj, const int flags)
+{
+	const struct ast_bucket_file *object;
+	const char *key;
+
+	switch (flags & OBJ_SEARCH_MASK) {
+	case OBJ_SEARCH_KEY:
+		key = obj;
+		break;
+	case OBJ_SEARCH_OBJECT:
+		object = obj;
+		key = ast_sorcery_object_get_id(object);
+		break;
+	default:
+		/* Hash can only work on something with a full key */
+		ast_assert(0);
+		return 0;
+	}
+	return ast_str_hash(key);
+}
+
+/*!
+ * \internal
+ * \brief Comparison function for file metadata
+ */
+static int media_cache_cmp(void *obj, void *arg, int flags)
+{
+	struct ast_bucket_file *left = obj;
+	struct ast_bucket_file *right = arg;
+	const char *right_key = arg;
+	int cmp;
+
+	switch (flags & OBJ_SEARCH_MASK) {
+	case OBJ_SEARCH_OBJECT:
+		right_key = ast_sorcery_object_get_id(right);
+		/* Fall through */
+	case OBJ_SEARCH_KEY:
+		cmp = strcmp(ast_sorcery_object_get_id(left), right_key);
+		break;
+	case OBJ_SEARCH_PARTIAL_KEY:
+		cmp = strncmp(ast_sorcery_object_get_id(left), right_key, strlen(right_key));
+		break;
+	default:
+		ast_assert(0);
+		cmp = 0;
+		break;
+	}
+
+	return cmp ? 0 : CMP_MATCH | CMP_STOP;
+}
+
+
+int ast_media_cache_exists(const char *uri)
+{
+	struct ast_bucket_file *bucket_file;
+
+	if (ast_strlen_zero(uri)) {
+		return 0;
+	}
+
+	bucket_file = ao2_find(media_cache, uri, OBJ_SEARCH_KEY);
+	if (bucket_file) {
+		ao2_ref(bucket_file, -1);
+		return 1;
+	}
+
+	/* Check to see if any bucket implementation could return this item */
+	bucket_file = ast_bucket_file_retrieve(uri);
+	if (bucket_file) {
+		ao2_ref(bucket_file, -1);
+		return 1;
+	}
+
+	return 0;
+}
+
+/*!
+ * \internal
+ * \brief Sync \c bucket_file metadata to the AstDB
+ */
+static int metadata_sync_to_astdb(void *obj, void *arg, int flags)
+{
+	struct ast_bucket_metadata *metadata = obj;
+	const char *hash = arg;
+
+	ast_db_put(hash, metadata->name, metadata->value);
+
+	return 0;
+}
+
+/*!
+ * \internal
+ * \brief Sync a media cache item to the AstDB
+ * \param bucket_file The \c ast_bucket_file media cache item to sync
+ */
+static void media_cache_item_sync_to_astdb(struct ast_bucket_file *bucket_file)
+{
+	char hash[41]; /* 40 character SHA1 hash */
+
+	ast_sha1_hash(hash, ast_sorcery_object_get_id(bucket_file));
+	if (ast_db_put(AST_DB_FAMILY, ast_sorcery_object_get_id(bucket_file), hash)) {
+		return;
+	}
+
+	ast_db_put(hash, "path", bucket_file->path);
+	ast_bucket_file_metadata_callback(bucket_file, metadata_sync_to_astdb, hash);
+}
+
+/*!
+ * \internal
+ * \brief Delete a media cache item from the AstDB
+ * \param bucket_file The \c ast_bucket_file media cache item to delete
+ */
+static void media_cache_item_del_from_astdb(struct ast_bucket_file *bucket_file)
+{
+	char *hash_value;
+
+	if (ast_db_get_allocated(AST_DB_FAMILY, ast_sorcery_object_get_id(bucket_file), &hash_value)) {
+		return;
+	}
+
+	ast_db_deltree(hash_value, NULL);
+	ast_db_del(AST_DB_FAMILY, hash_value);
+	ast_free(hash_value);
+}
+
+/*!
+ * \internal
+ * \brief Update the name of the file backing a \c bucket_file
+ * \param preferred_file_name The preferred name of the backing file
+ */
+static void bucket_file_update_path(struct ast_bucket_file *bucket_file,
+	const char *preferred_file_name)
+{
+	if (ast_strlen_zero(preferred_file_name)) {
+		return;
+	}
+
+	if (!strcmp(bucket_file->path, preferred_file_name)) {
+		return;
+	}
+
+	rename(bucket_file->path, preferred_file_name);
+	ast_copy_string(bucket_file->path, preferred_file_name,
+		sizeof(bucket_file->path));
+}
+
+int ast_media_cache_retrieve(const char *uri, const char *preferred_file_name,
+	char *file_path, size_t len)
+{
+	struct ast_bucket_file *bucket_file;
+	SCOPED_AO2LOCK(media_lock, media_cache);
+
+	if (ast_strlen_zero(uri)) {
+		return -1;
+	}
+
+	/* First, retrieve from the ao2 cache here. If we find a bucket_file
+	 * matching the requested URI, ask the appropriate backend if it is
+	 * stale. If not; return it.
+	 */
+	bucket_file = ao2_find(media_cache, uri, OBJ_SEARCH_KEY | OBJ_NOLOCK);
+	if (bucket_file) {
+		if (!ast_bucket_file_is_stale(bucket_file)) {
+			ast_copy_string(file_path, bucket_file->path, len);
+			ao2_ref(bucket_file, -1);
+			return 0;
+		}
+
+		/* Stale! Drop the ref, as we're going to retrieve it next. */
+		ao2_ref(bucket_file, -1);
+	}
+
+	/* Either this is new or the resource is stale; do a full retrieve
+	 * from the appropriate bucket_file backend
+	 */
+	bucket_file = ast_bucket_file_retrieve(uri);
+	if (!bucket_file) {
+		ast_log(LOG_WARNING, "Failed to obtain media at '%s'\n", uri);
+		return -1;
+	}
+
+	/* We can manipulate the 'immutable' bucket_file here, as we haven't
+	 * let anyone know of its existence yet
+	 */
+	bucket_file_update_path(bucket_file, preferred_file_name);
+	media_cache_item_sync_to_astdb(bucket_file);
+	ast_copy_string(file_path, bucket_file->path, len);
+	ao2_link_flags(media_cache, bucket_file, OBJ_NOLOCK);
+	ao2_ref(bucket_file, -1);
+
+	return 0;
+}
+
+int ast_media_cache_retrieve_metadata(const char *uri, const char *key,
+	char *value, size_t len)
+{
+	struct ast_bucket_file *bucket_file;
+	struct ast_bucket_metadata *metadata;
+
+	if (ast_strlen_zero(uri) || ast_strlen_zero(key) || !value) {
+		return -1;
+	}
+
+	bucket_file = ao2_find(media_cache, uri, OBJ_SEARCH_KEY);
+	if (!bucket_file) {
+		return -1;
+	}
+
+	metadata = ao2_find(bucket_file->metadata, key, OBJ_SEARCH_KEY);
+	if (!metadata) {
+		ao2_ref(bucket_file, -1);
+		return -1;
+	}
+	ast_copy_string(value, metadata->value, len);
+
+	ao2_ref(metadata, -1);
+	ao2_ref(bucket_file, -1);
+	return 0;
+}
+
+int ast_media_cache_create_or_update(const char *uri, const char *file_path,
+	struct ast_variable *metadata)
+{
+	struct ast_bucket_file *bucket_file;
+	struct ast_variable *it_metadata;
+	struct stat st;
+	char tmp[128];
+	char *ext;
+	char *file_path_ptr;
+	int created = 0;
+	SCOPED_AO2LOCK(media_lock, media_cache);
+
+	if (ast_strlen_zero(file_path) || ast_strlen_zero(uri)) {
+		return -1;
+	}
+	file_path_ptr = ast_strdupa(file_path);
+
+	if (stat(file_path, &st)) {
+		ast_log(LOG_WARNING, "Unable to obtain information for file %s for URI %s\n",
+			file_path, uri);
+		return -1;
+	}
+
+	bucket_file = ao2_find(media_cache, uri, OBJ_SEARCH_KEY | OBJ_NOLOCK);
+	if (bucket_file) {
+		struct ast_bucket_file *clone;
+
+		clone = ast_bucket_file_clone(bucket_file);
+		if (!clone) {
+			ao2_ref(bucket_file, -1);
+			return -1;
+		}
+
+		/* Remove the old bucket_file. We'll replace it if we succeed below. */
+		ao2_unlink_flags(media_cache, bucket_file, OBJ_NOLOCK);
+		ao2_ref(bucket_file, -1);
+
+		bucket_file = clone;
+	} else {
+		bucket_file = ast_bucket_file_alloc(uri);
+		if (!bucket_file) {
+			ast_log(LOG_WARNING, "Failed to create file storage for %s and %s\n",
+				uri, file_path);
+			return -1;
+		}
+		created = 1;
+	}
+
+	strcpy(bucket_file->path, file_path);
+	bucket_file->created.tv_sec = st.st_ctime;
+	bucket_file->modified.tv_sec = st.st_mtime;
+
+	snprintf(tmp, sizeof(tmp), "%ld", (long)st.st_atime);
+	ast_bucket_file_metadata_set(bucket_file, "accessed", tmp);
+
+	snprintf(tmp, sizeof(tmp), "%jd", (intmax_t)st.st_size);
+	ast_bucket_file_metadata_set(bucket_file, "size", tmp);
+
+	ext = strrchr(file_path_ptr, '.');
+	if (ext) {
+		ast_bucket_file_metadata_set(bucket_file, "ext", ext + 1);
+	}
+
+	for (it_metadata = metadata; it_metadata; it_metadata = it_metadata->next) {
+		ast_bucket_file_metadata_set(bucket_file, it_metadata->name, it_metadata->value);
+	}
+
+	if (created && ast_bucket_file_create(bucket_file)) {
+		ast_log(LOG_WARNING, "Failed to create media for %s\n", uri);
+		ao2_ref(bucket_file, -1);
+		return -1;
+	}
+	media_cache_item_sync_to_astdb(bucket_file);
+
+	ao2_link_flags(media_cache, bucket_file, OBJ_NOLOCK);
+	ao2_ref(bucket_file, -1);
+	return 0;
+}
+
+int ast_media_cache_delete(const char *uri)
+{
+	struct ast_bucket_file *bucket_file;
+	int res;
+
+	if (ast_strlen_zero(uri)) {
+		return -1;
+	}
+
+	bucket_file = ao2_find(media_cache, uri, OBJ_SEARCH_KEY | OBJ_UNLINK);
+	if (!bucket_file) {
+		return -1;
+	}
+
+	res = ast_bucket_file_delete(bucket_file);
+	media_cache_item_del_from_astdb(bucket_file);
+
+	ao2_ref(bucket_file, -1);
+
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Shutdown the media cache
+ */
+static void media_cache_shutdown(void)
+{
+	ao2_ref(media_cache, -1);
+	media_cache = NULL;
+}
+
+/*!
+ * \internal
+ * \brief Remove a media cache item from the AstDB
+ * \param uri The unique URI that represents the item in the cache
+ * \param hash The hash key for the item in the AstDB
+ */
+static void media_cache_remove_from_astdb(const char *uri, const char *hash)
+{
+	ast_db_del(AST_DB_FAMILY, uri + AST_DB_FAMILY_LEN);
+	ast_db_deltree(hash, NULL);
+}
+
+/*!
+ * \internal
+ * \brief Create an item in the media cache from entries in the AstDB
+ * \param uri The unique URI that represents the item in the cache
+ * \param hash The hash key for the item in the AstDB
+ * \retval 0 success
+ * \retval -1 failure
+ */
+static int media_cache_item_populate_from_astdb(const char *uri, const char *hash)
+{
+	struct ast_bucket_file *bucket_file;
+	struct ast_db_entry *db_tree;
+	struct ast_db_entry *db_entry;
+	struct stat st;
+
+	bucket_file = ast_bucket_file_alloc(uri);
+	if (!bucket_file) {
+		return -1;
+	}
+
+	db_tree = ast_db_gettree(hash, NULL);
+	for (db_entry = db_tree; db_entry; db_entry = db_entry->next) {
+		const char *key = strchr(db_entry->key + 1, '/');
+
+		if (ast_strlen_zero(key)) {
+			continue;
+		}
+		key++;
+
+		if (!strcasecmp(key, "path")) {
+			strcpy(bucket_file->path, db_entry->data);
+
+			if (stat(bucket_file->path, &st)) {
+				ast_log(LOG_WARNING, "Unable to obtain information for file %s for URI %s\n",
+					bucket_file->path, uri);
+				ao2_ref(bucket_file, -1);
+				ast_db_freetree(db_tree);
+				return -1;
+			}
+		} else {
+			ast_bucket_file_metadata_set(bucket_file, key, db_entry->data);
+		}
+	}
+	ast_db_freetree(db_tree);
+
+	if (ast_strlen_zero(bucket_file->path)) {
+		ao2_ref(bucket_file, -1);
+		ast_log(LOG_WARNING, "Failed to restore media cache item for '%s' from AstDB: no 'path' specified\n",
+			uri);
+		return -1;
+	}
+
+	ao2_link(media_cache, bucket_file);
+	ao2_ref(bucket_file, -1);
+
+	return 0;
+}
+
+/*!
+ * \internal
+ * \brief Populate the media cache from entries in the AstDB
+ */
+static void media_cache_populate_from_astdb(void)
+{
+	struct ast_db_entry *db_entry;
+	struct ast_db_entry *db_tree;
+
+	db_tree = ast_db_gettree(AST_DB_FAMILY, NULL);
+	for (db_entry = db_tree; db_entry; db_entry = db_entry->next) {
+		if (media_cache_item_populate_from_astdb(db_entry->key + AST_DB_FAMILY_LEN, db_entry->data)) {
+			media_cache_remove_from_astdb(db_entry->key, db_entry->data);
+		}
+	}
+	ast_db_freetree(db_tree);
+}
+
+int ast_media_cache_init(void)
+{
+	ast_register_atexit(media_cache_shutdown);
+
+	media_cache = ao2_container_alloc_options(AO2_ALLOC_OPT_LOCK_RWLOCK, AO2_BUCKETS,
+		media_cache_hash, media_cache_cmp);
+	if (!media_cache) {
+		return -1;
+	}
+
+	media_cache_populate_from_astdb();
+
+	return 0;
+}