Add a 'consensus cache' type on top of storagedir. (4404dc57) · Commits · Benjamin J. Thompson / Tor

src/or/conscache.c

0 → 100644

+487 −0

Original line number	Diff line number	Diff line
		/* Copyright (c) 2017, The Tor Project, Inc. */
		/* See LICENSE for licensing information */

		#include "or.h"

		#include "config.h"
		#include "conscache.h"
		#include "storagedir.h"

		#define CCE_MAGIC 0x17162253

		/**
		* A consensus_cache_entry_t is a reference-counted handle to an
		* item in a consensus_cache_t. It can be mmapped into RAM, or not,
		* depending whether it's currently in use.
		*/
		struct consensus_cache_entry_t {
		uint32_t magic; /*< Must be set to CCE_MAGIC /
		int32_t refcnt; /*< Reference count. /
		unsigned can_remove : 1; /*< If true, we want to delete this file. /
		/** If true, we intend to unmap this file as soon as we're done with it. */
		unsigned release_aggressively : 1;

		/** Filename for this object within the storage_dir_t */
		char *fname;
		/** Labels associated with this object. Immutable once the object
		* is created. */
		config_line_t *labels;
		/** Pointer to the cache that includes this entry (if any). */
		consensus_cache_t *in_cache;

		/** Since what time has this object been mapped into RAM, but with the cache
		* being the only having a reference to it? */
		time_t unused_since;
		/** mmaped contents of the underlying file. May be NULL */
		tor_mmap_t *map;
		/** Length of the body within <b>map</b>. */
		size_t bodylen;
		/** Pointer to the body within <b>map</b>. */
		const uint8_t *body;
		};

		/**
		* A consensus_cache_t holds a directory full of labeled items.
		*/
		struct consensus_cache_t {
		/** Underling storage_dir_t to handle persistence */
		storage_dir_t *dir;
		/** List of all the entries in the directory. */
		smartlist_t *entries;
		};

		static void consensus_cache_clear(consensus_cache_t *cache);
		static void consensus_cache_rescan(consensus_cache_t *);
		static void consensus_cache_entry_map(consensus_cache_t *,
		consensus_cache_entry_t *);
		static void consensus_cache_entry_unmap(consensus_cache_entry_t *ent);

		/**
		* Helper: Open a consensus cache in subdirectory <b>subdir</b> of the
		* data directory, to hold up to <b>max_entries</b> of data.
		*/
		consensus_cache_t *
		consensus_cache_open(const char *subdir, int max_entries)
		{
		consensus_cache_t *cache = tor_malloc_zero(sizeof(consensus_cache_t));
		char *directory = get_datadir_fname(subdir);
		cache->dir = storage_dir_new(directory, max_entries);
		tor_free(directory);
		if (!cache->dir) {
		tor_free(cache);
		return NULL;
		}

		consensus_cache_rescan(cache);
		return cache;
		}

		/**
		* Helper: clear all entries from <b>cache</b> (but do not delete
		* any that aren't marked for removal
		*/
		static void
		consensus_cache_clear(consensus_cache_t *cache)
		{
		consensus_cache_delete_pending(cache);

		SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
		ent->in_cache = NULL;
		consensus_cache_entry_decref(ent);
		} SMARTLIST_FOREACH_END(ent);
		smartlist_free(cache->entries);
		cache->entries = NULL;
		}

		/**
		* Drop all storage held by <b>cache</b>.
		*/
		void
		consensus_cache_free(consensus_cache_t *cache)
		{
		if (! cache)
		return;

		if (cache->entries) {
		consensus_cache_clear(cache);
		}
		storage_dir_free(cache->dir);
		tor_free(cache);
		}

		/**
		* Write <b>datalen</b> bytes of data at <b>data</b> into the <b>cache</b>,
		* labeling that data with <b>labels</b>. On failure, return NULL. On
		* success, return a newly created consensus_cache_entry_t.
		*
		* The returned value will be owned by the cache, and you will have a
		* reference to it. Call consensus_cache_entry_decref() when you are
		* done with it.
		*
		* The provided <b>labels</b> MUST have distinct keys: if they don't,
		* this API does not specify which values (if any) for the duplicate keys
		* will be considered.
		*/
		consensus_cache_entry_t *
		consensus_cache_add(consensus_cache_t *cache,
		const config_line_t *labels,
		const uint8_t *data,
		size_t datalen)
		{
		char *fname = NULL;
		int r = storage_dir_save_labeled_to_file(cache->dir,
		labels, data, datalen, &fname);
		if (r < 0 \|\| fname == NULL) {
		return NULL;
		}
		consensus_cache_entry_t *ent =
		tor_malloc_zero(sizeof(consensus_cache_entry_t));
		ent->magic = CCE_MAGIC;
		ent->fname = fname;
		ent->labels = config_lines_dup(labels);
		ent->in_cache = cache;
		ent->unused_since = TIME_MAX;
		smartlist_add(cache->entries, ent);
		/* Start the reference count at 2: the caller owns one copy, and the
		* cache owns another.
		*/
		ent->refcnt = 2;

		return ent;
		}

		/**
		* Given a <b>cache</b>, return some entry for which <b>key</b>=<b>value</b>.
		* Return NULL if no such entry exists.
		*
		* Does not adjust reference counts.
		*/
		consensus_cache_entry_t *
		consensus_cache_find_first(consensus_cache_t *cache,
		const char *key,
		const char *value)
		{
		smartlist_t *tmp = smartlist_new();
		consensus_cache_find_all(tmp, cache, key, value);
		consensus_cache_entry_t *ent = NULL;
		if (smartlist_len(tmp))
		ent = smartlist_get(tmp, 0);
		smartlist_free(tmp);
		return ent;
		}

		/**
		* Given a <b>cache</b>, add every entry to <b>out<b> for which
		* <b>key</b>=<b>value</b>. If <b>key</b> is NULL, add every entry.
		*
		* Does not adjust reference counts.
		*/
		void
		consensus_cache_find_all(smartlist_t *out,
		consensus_cache_t *cache,
		const char *key,
		const char *value)
		{
		if (! key) {
		smartlist_add_all(out, cache->entries);
		return;
		}

		SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
		const char *found_val = consensus_cache_entry_get_value(ent, key);
		if (found_val && !strcmp(value, found_val)) {
		smartlist_add(out, ent);
		}
		} SMARTLIST_FOREACH_END(ent);
		}

		/**
		* Given a list of consensus_cache_entry_t, remove all those entries
		* that do not have <b>key</b>=<b>value</b> in their labels.
		*
		* Does not adjust reference counts.
		*/
		void
		consensus_cache_filter_list(smartlist_t *lst,
		const char *key,
		const char *value)
		{
		if (BUG(lst == NULL))
		return; // LCOV_EXCL_LINE
		if (key == NULL)
		return;
		SMARTLIST_FOREACH_BEGIN(lst, consensus_cache_entry_t *, ent) {
		const char *found_val = consensus_cache_entry_get_value(ent, key);
		if (! found_val \|\| strcmp(value, found_val)) {
		SMARTLIST_DEL_CURRENT(lst, ent);
		}
		} SMARTLIST_FOREACH_END(ent);
		}

		/**
		* If <b>ent</b> has a label with the given <b>key</b>, return its
		* value. Otherwise return NULL.
		*
		* The return value is only guaranteed to be valid for as long as you
		* hold a reference to <b>ent</b>.
		*/
		const char *
		consensus_cache_entry_get_value(const consensus_cache_entry_t *ent,
		const char *key)
		{
		const config_line_t *match = config_line_find(ent->labels, key);
		if (match)
		return match->value;
		else
		return NULL;
		}

		/**
		* Return a pointer to the labels in <b>ent</b>.
		*
		* This pointer is only guaranteed to be valid for as long as you
		* hold a reference to <b>ent</b>.
		*/
		const config_line_t *
		consensus_cache_entry_get_labels(const consensus_cache_entry_t *ent)
		{
		return ent->labels;
		}

		/**
		* Increase the reference count of <b>ent</b>.
		*/
		void
		consensus_cache_entry_incref(consensus_cache_entry_t *ent)
		{
		if (BUG(ent->magic != CCE_MAGIC))
		return; // LCOV_EXCL_LINE
		++ent->refcnt;
		ent->unused_since = TIME_MAX;
		}

		/**
		* Release a reference held to <b>ent</b>.
		*
		* If it was the last reference, ent will be freed. Therefore, you must not
		* use <b>ent</b> after calling this function.
		*/
		void
		consensus_cache_entry_decref(consensus_cache_entry_t *ent)
		{
		if (! ent)
		return;
		if (BUG(ent->refcnt <= 0))
		return; // LCOV_EXCL_LINE
		if (BUG(ent->magic != CCE_MAGIC))
		return; // LCOV_EXCL_LINE

		--ent->refcnt;

		if (ent->refcnt == 1 && ent->in_cache) {
		/* Only the cache has a reference: we don't need to keep the file
		* mapped */
		if (ent->map) {
		if (ent->release_aggressively) {
		consensus_cache_entry_unmap(ent);
		} else {
		ent->unused_since = approx_time();
		}
		}
		return;
		}

		if (ent->refcnt > 0)
		return;

		/* Refcount is zero; we can free it. */
		if (ent->map) {
		consensus_cache_entry_unmap(ent);
		}
		tor_free(ent->fname);
		config_free_lines(ent->labels);
		memwipe(ent, 0, sizeof(consensus_cache_entry_t));
		tor_free(ent);
		}

		/**
		* Mark <b>ent</b> for deletion from the cache. Deletion will not occur
		* until the cache is the only place that holds a reference to <b>ent</b>.
		*/
		void
		consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent)
		{
		ent->can_remove = 1;
		}

		/**
		* Mark <b>ent</b> as the kind of entry that we don't need to keep mmap'd for
		* any longer than we're actually using it.
		*/
		void
		consensus_cache_entry_mark_for_aggressive_release(consensus_cache_entry_t *ent)
		{
		ent->release_aggressively = 1;
		}

		/**
		* Try to read the body of <b>ent</b> into memory if it isn't already
		* loaded. On success, set <b>body_out</b> to the body, <b>sz_out</b>
		* to its size, and return 0. On failure return -1.
		*
		* The resulting body pointer will only be valid for as long as you
		* hold a reference to <b>ent</b>.
		*/
		int
		consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
		const uint8_t **body_out,
		size_t *sz_out)
		{
		if (BUG(ent->magic != CCE_MAGIC))
		return -1; // LCOV_EXCL_LINE

		if (! ent->map) {
		if (! ent->in_cache)
		return -1;

		consensus_cache_entry_map((consensus_cache_t *)ent->in_cache,
		(consensus_cache_entry_t *)ent);
		if (! ent->map) {
		return -1;
		}
		}

		*body_out = ent->body;
		*sz_out = ent->bodylen;
		return 0;
		}

		/**
		* Unmap every mmap'd element of <b>cache</b> that has been unused
		* since <b>cutoff</b>.
		*/
		void
		consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff)
		{
		SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
		tor_assert_nonfatal(ent->in_cache == cache);
		if (ent->refcnt > 1 \|\| BUG(ent->in_cache == NULL)) {
		/* Somebody is using this entry right now */
		continue;
		}
		if (ent->unused_since > cutoff) {
		/* Has been unused only for a little while */
		continue;
		}
		if (ent->map == NULL) {
		/* Not actually mapped. */
		continue;
		}
		consensus_cache_entry_unmap(ent);
		} SMARTLIST_FOREACH_END(ent);
		}

		/**
		* Delete every element of <b>cache</b> has been marked with
		* consensus_cache_entry_mark_for_removal, and which is not in use except by
		* the cache.
		*/
		void
		consensus_cache_delete_pending(consensus_cache_t *cache)
		{
		SMARTLIST_FOREACH_BEGIN(cache->entries, consensus_cache_entry_t *, ent) {
		tor_assert_nonfatal(ent->in_cache == cache);
		if (ent->refcnt > 1 \|\| BUG(ent->in_cache == NULL)) {
		/* Somebody is using this entry right now */
		continue;
		}
		if (ent->can_remove == 0) {
		/* Don't want to delete this. */
		continue;
		}
		if (BUG(ent->refcnt <= 0)) {
		continue; // LCOV_EXCL_LINE
		}

		SMARTLIST_DEL_CURRENT(cache->entries, ent);
		ent->in_cache = NULL;
		char fname = tor_strdup(ent->fname); / save a copy */
		consensus_cache_entry_decref(ent);
		storage_dir_remove_file(cache->dir, fname);
		tor_free(fname);
		} SMARTLIST_FOREACH_END(ent);
		}

		/**
		* Internal helper: rescan <b>cache</b> and rebuild its list of entries.
		*/
		static void
		consensus_cache_rescan(consensus_cache_t *cache)
		{
		if (cache->entries) {
		consensus_cache_clear(cache);
		}

		cache->entries = smartlist_new();
		const smartlist_t *fnames = storage_dir_list(cache->dir);
		SMARTLIST_FOREACH_BEGIN(fnames, const char *, fname) {
		tor_mmap_t *map = NULL;
		config_line_t *labels = NULL;
		const uint8_t *body;
		size_t bodylen;
		map = storage_dir_map_labeled(cache->dir, fname,
		&labels, &body, &bodylen);
		if (! map) {
		/* Can't load this; continue */
		log_warn(LD_FS, "Unable to map file %s from consensus cache: %s",
		escaped(fname), strerror(errno));
		continue;
		}
		consensus_cache_entry_t *ent =
		tor_malloc_zero(sizeof(consensus_cache_entry_t));
		ent->magic = CCE_MAGIC;
		ent->fname = tor_strdup(fname);
		ent->labels = labels;
		ent->refcnt = 1;
		ent->in_cache = cache;
		ent->unused_since = TIME_MAX;
		smartlist_add(cache->entries, ent);
		tor_munmap_file(map); /* don't actually need to keep this around */
		} SMARTLIST_FOREACH_END(fname);
		}

		/**
		* Make sure that <b>ent</b> is mapped into RAM.
		*/
		static void
		consensus_cache_entry_map(consensus_cache_t *cache,
		consensus_cache_entry_t *ent)
		{
		if (ent->map)
		return;

		ent->map = storage_dir_map_labeled(cache->dir, ent->fname,
		NULL, &ent->body, &ent->bodylen);
		ent->unused_since = TIME_MAX;
		}

		/**
		* Unmap <b>ent</b> from RAM.
		*
		* Do not call this if something other than the cache is holding a reference
		* to <b>ent</b>
		*/
		static void
		consensus_cache_entry_unmap(consensus_cache_entry_t *ent)
		{
		ent->unused_since = TIME_MAX;
		if (!ent->map)
		return;

		tor_munmap_file(ent->map);
		ent->map = NULL;
		ent->body = NULL;
		ent->bodylen = 0;
		ent->unused_since = TIME_MAX;
		}

src/or/conscache.h

0 → 100644

+48 −0

Original line number	Diff line number	Diff line
		/* Copyright (c) 2017, The Tor Project, Inc. */
		/* See LICENSE for licensing information */

		#ifndef TOR_CONSCACHE_H
		#define TOR_CONSCACHE_H

		typedef struct consensus_cache_entry_t consensus_cache_entry_t;
		typedef struct consensus_cache_t consensus_cache_t;

		consensus_cache_t consensus_cache_open(const char subdir, int max_entries);
		void consensus_cache_free(consensus_cache_t *cache);
		void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
		void consensus_cache_delete_pending(consensus_cache_t *cache);
		consensus_cache_entry_t consensus_cache_add(consensus_cache_t cache,
		const config_line_t *labels,
		const uint8_t *data,
		size_t datalen);

		consensus_cache_entry_t *consensus_cache_find_first(
		consensus_cache_t *cache,
		const char *key,
		const char *value);

		void consensus_cache_find_all(smartlist_t *out,
		consensus_cache_t *cache,
		const char *key,
		const char *value);
		void consensus_cache_filter_list(smartlist_t *lst,
		const char *key,
		const char *value);

		const char consensus_cache_entry_get_value(const consensus_cache_entry_t ent,
		const char *key);
		const config_line_t *consensus_cache_entry_get_labels(
		const consensus_cache_entry_t *ent);

		void consensus_cache_entry_incref(consensus_cache_entry_t *ent);
		void consensus_cache_entry_decref(consensus_cache_entry_t *ent);

		void consensus_cache_entry_mark_for_removal(consensus_cache_entry_t *ent);
		void consensus_cache_entry_mark_for_aggressive_release(
		consensus_cache_entry_t *ent);
		int consensus_cache_entry_get_body(const consensus_cache_entry_t *ent,
		const uint8_t **body_out,
		size_t *sz_out);

		#endif

src/or/include.am

+2 −0

Original line number	Diff line number	Diff line
		@@ -36,6 +36,7 @@ LIBTOR_A_SOURCES = \
		src/or/connection.c \
		src/or/connection_edge.c \
		src/or/connection_or.c \
		src/or/conscache.c \
		src/or/consdiff.c \
		src/or/control.c \
		src/or/cpuworker.c \
		@@ -152,6 +153,7 @@ ORHEADERS = \
		src/or/connection.h \
		src/or/connection_edge.h \
		src/or/connection_or.h \
		src/or/conscache.h \
		src/or/consdiff.h \
		src/or/control.h \
		src/or/cpuworker.h \