consensus diff manager functions More...

#include "core/or/or.h"
#include "app/config/config.h"
#include "feature/dircache/conscache.h"
#include "feature/dircommon/consdiff.h"
#include "feature/dircache/consdiffmgr.h"
#include "core/mainloop/cpuworker.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/dirparse/ns_parse.h"
#include "lib/evloop/compat_libevent.h"
#include "lib/evloop/workqueue.h"
#include "lib/compress/compress.h"
#include "lib/encoding/confline.h"
#include "feature/nodelist/networkstatus_st.h"
#include "feature/nodelist/networkstatus_voter_info_st.h"

Data Structures
struct	cdm_diff_t

struct	compressed_result_t

struct	consensus_diff_worker_job_t

struct	consensus_compress_worker_job_t

Macros
#define	CONSDIFFMGR_PRIVATE

#define	DOCTYPE_CONSENSUS "consensus"

#define	DOCTYPE_CONSENSUS_DIFF "consensus-diff"

#define	RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD

#define	CACHE_MAX_NUM 128

#define	cdm_diff_free(diff) FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))

#define	consensus_diff_worker_job_free(job)

#define	consensus_compress_worker_job_free(job)


#define	LABEL_DOCTYPE "document-type"

#define	LABEL_VALID_AFTER "consensus-valid-after"

#define	LABEL_FRESH_UNTIL "consensus-fresh-until"

#define	LABEL_VALID_UNTIL "consensus-valid-until"

#define	LABEL_SIGNATORIES "consensus-signatories"

#define	LABEL_SHA3_DIGEST "sha3-digest"

#define	LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"

#define	LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"

#define	LABEL_FLAVOR "consensus-flavor"

#define	LABEL_FROM_SHA3_DIGEST "from-sha3-digest"

#define	LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"

#define	LABEL_FROM_VALID_AFTER "from-valid-after"

#define	LABEL_COMPRESSION_TYPE "compression"

Enumerations
enum	cdm_diff_status_t { CDM_DIFF_PRESENT =1 , CDM_DIFF_IN_PROGRESS =2 , CDM_DIFF_ERROR =3 }

Functions
static void	consdiffmgr_rescan_cb (mainloop_event_t ev, void arg)

static void	mark_cdm_cache_dirty (void)

STATIC unsigned	n_diff_compression_methods (void)

STATIC unsigned	n_consensus_compression_methods (void)

static	HT_HEAD (cdm_diff_ht, cdm_diff_t)

static int	consdiffmgr_ensure_space_for_files (int n)

static int	consensus_queue_compression_work (const char consensus, size_t consensus_len, const networkstatus_t as_parsed)

static int	consensus_diff_queue_diff_work (consensus_cache_entry_t diff_from, consensus_cache_entry_t diff_to)

static void	consdiffmgr_set_cache_flags (void)

static unsigned	cdm_diff_hash (const cdm_diff_t *diff)

static int	cdm_diff_eq (const cdm_diff_t diff1, const cdm_diff_t diff2)

	HT_PROTOTYPE (cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)

	HT_GENERATE2 (cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq, 0.6, tor_reallocarray, tor_free_)

static void	cdm_diff_free_ (cdm_diff_t *diff)

static cdm_diff_t *	cdm_diff_new (consensus_flavor_t flav, const uint8_t from_sha3, const uint8_t target_sha3, compress_method_t method)

static int	cdm_diff_ht_check_and_note_pending (consensus_flavor_t flav, const uint8_t from_sha3, const uint8_t target_sha3)

static void	cdm_diff_ht_set_status (consensus_flavor_t flav, const uint8_t from_sha3, const uint8_t to_sha3, compress_method_t method, int status, consensus_cache_entry_handle_t *handle)

static void	cdm_diff_ht_purge (consensus_flavor_t flav, const uint8_t *unless_target_sha3_matches)

static void	cdm_cache_init (void)

STATIC consensus_cache_t *	cdm_cache_get (void)

static void	cdm_labels_prepend_sha3 (config_line_t *labels, const char label, const uint8_t *body, size_t bodylen)

STATIC int	cdm_entry_get_sha3_value (uint8_t digest_out, consensus_cache_entry_t ent, const char *label)

STATIC consensus_cache_entry_t *	cdm_cache_lookup_consensus (consensus_flavor_t flavor, time_t valid_after)

static int32_t	get_max_age_to_cache (void)

int	consdiffmgr_add_consensus (const char consensus, size_t consensus_len, const networkstatus_t as_parsed)

static int	compare_by_valid_after_ (const void a, const void b)

static consensus_cache_entry_t *	sort_and_find_most_recent (smartlist_t *lst)

static int	consensus_compression_method_pos (compress_method_t method)

consdiff_status_t	consdiffmgr_find_consensus (struct consensus_cache_entry_t **entry_out, consensus_flavor_t flavor, compress_method_t method)

consdiff_status_t	consdiffmgr_find_diff_from (consensus_cache_entry_t *entry_out, consensus_flavor_t flavor, int digest_type, const uint8_t digest, size_t digestlen, compress_method_t method)

int	consdiffmgr_cleanup (void)

void	consdiffmgr_configure (const consdiff_cfg_t *cfg)

int	consdiffmgr_register_with_sandbox (struct sandbox_cfg_elem_t **cfg)

int	consdiffmgr_validate (void)

static void	consdiffmgr_rescan_flavor_ (consensus_flavor_t flavor)

static void	consdiffmgr_consensus_load (void)

static void	consdiffmgr_diffs_load (void)

void	consdiffmgr_rescan (void)

static int	compare_by_staleness_ (const void a, const void b)

void	consdiffmgr_free_all (void)

static int	compress_multiple (compressed_result_t results_out, int n_methods, const compress_method_t methods, const uint8_t input, size_t len, const config_line_t labels_in)

static cdm_diff_status_t	store_multiple (consensus_cache_entry_handle_t *handles_out, int n, const compress_method_t methods, const compressed_result_t results, const char description)

STATIC int	uncompress_or_set_ptr (const char *out, size_t outlen, char *owned_out, consensus_cache_entry_t ent)

static workqueue_reply_t	consensus_diff_worker_threadfn (void state_, void work_)

static void	consensus_diff_worker_job_free_ (consensus_diff_worker_job_t *job)

static void	consensus_diff_worker_replyfn (void *work_)

static void	consensus_compress_worker_job_free_ (consensus_compress_worker_job_t *job)

static workqueue_reply_t	consensus_compress_worker_threadfn (void state_, void work_)

static void	consensus_compress_worker_replyfn (void *work_)

void	consdiffmgr_enable_background_compression (void)

int	consensus_cache_entry_get_voter_id_digests (const consensus_cache_entry_t ent, smartlist_t out)

int	consensus_cache_entry_get_fresh_until (const consensus_cache_entry_t ent, time_t out)

int	consensus_cache_entry_get_valid_until (const consensus_cache_entry_t ent, time_t out)

int	consensus_cache_entry_get_valid_after (const consensus_cache_entry_t ent, time_t out)

Variables
static consensus_cache_t *	cons_diff_cache = NULL

static int	cdm_cache_dirty = 0

static int	cdm_cache_loaded = 0

static const compress_method_t	compress_diffs_with []

static mainloop_event_t *	consdiffmgr_rescan_ev = NULL

static const compress_method_t	compress_consensus_with []

static consensus_cache_entry_handle_t *	latest_consensus [N_CONSENSUS_FLAVORS][ARRAY_LENGTH(compress_consensus_with)]

static int	background_compression = 0

Detailed Description

consensus diff manager functions

This module is run by directory authorities and caches in order to remember a number of past consensus documents, and to generate and serve the diffs from those documents to the latest consensus.

Definition in file consdiffmgr.c.

Macro Definition Documentation

◆ cdm_diff_free

#define cdm_diff_free ( diff ) FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))

Definition at line 235 of file consdiffmgr.c.

◆ CONSDIFFMGR_PRIVATE

#define CONSDIFFMGR_PRIVATE

Definition at line 14 of file consdiffmgr.c.

◆ consensus_compress_worker_job_free

#define consensus_compress_worker_job_free ( job )

Value:

FREE_AND_NULL(consensus_compress_worker_job_t, \

consensus_compress_worker_job_free_, (job))

consensus_compress_worker_job_free_

static void consensus_compress_worker_job_free_(consensus_compress_worker_job_t *job)

Definition: consdiffmgr.c:1760

consensus_compress_worker_job_t

Definition: consdiffmgr.c:1744

Definition at line 1752 of file consdiffmgr.c.

◆ consensus_diff_worker_job_free

#define consensus_diff_worker_job_free ( job )

Value:

FREE_AND_NULL(consensus_diff_worker_job_t, \

consensus_diff_worker_job_free_, (job))

consensus_diff_worker_job_free_

static void consensus_diff_worker_job_free_(consensus_diff_worker_job_t *job)

Definition: consdiffmgr.c:1604

consensus_diff_worker_job_t

Definition: consdiffmgr.c:1400

Definition at line 1596 of file consdiffmgr.c.

◆ DOCTYPE_CONSENSUS

#define DOCTYPE_CONSENSUS "consensus"

Definition at line 69 of file consdiffmgr.c.

◆ DOCTYPE_CONSENSUS_DIFF

#define DOCTYPE_CONSENSUS_DIFF "consensus-diff"

Definition at line 70 of file consdiffmgr.c.

◆ LABEL_COMPRESSION_TYPE

#define LABEL_COMPRESSION_TYPE "compression"

Definition at line 66 of file consdiffmgr.c.

◆ LABEL_DOCTYPE

#define LABEL_DOCTYPE "document-type"

Labels to apply to items in the conscache object.

Definition at line 38 of file consdiffmgr.c.

◆ LABEL_FLAVOR

#define LABEL_FLAVOR "consensus-flavor"

Definition at line 58 of file consdiffmgr.c.

◆ LABEL_FRESH_UNTIL

#define LABEL_FRESH_UNTIL "consensus-fresh-until"

Definition at line 44 of file consdiffmgr.c.

◆ LABEL_FROM_SHA3_DIGEST

#define LABEL_FROM_SHA3_DIGEST "from-sha3-digest"

Definition at line 60 of file consdiffmgr.c.

◆ LABEL_FROM_VALID_AFTER

#define LABEL_FROM_VALID_AFTER "from-valid-after"

Definition at line 64 of file consdiffmgr.c.

◆ LABEL_SHA3_DIGEST

#define LABEL_SHA3_DIGEST "sha3-digest"

Definition at line 52 of file consdiffmgr.c.

◆ LABEL_SHA3_DIGEST_AS_SIGNED

#define LABEL_SHA3_DIGEST_AS_SIGNED "sha3-digest-as-signed"

Definition at line 56 of file consdiffmgr.c.

◆ LABEL_SHA3_DIGEST_UNCOMPRESSED

#define LABEL_SHA3_DIGEST_UNCOMPRESSED "sha3-digest-uncompressed"

Definition at line 54 of file consdiffmgr.c.

◆ LABEL_SIGNATORIES

#define LABEL_SIGNATORIES "consensus-signatories"

Definition at line 50 of file consdiffmgr.c.

◆ LABEL_TARGET_SHA3_DIGEST

#define LABEL_TARGET_SHA3_DIGEST "target-sha3-digest"

Definition at line 62 of file consdiffmgr.c.

◆ LABEL_VALID_AFTER

#define LABEL_VALID_AFTER "consensus-valid-after"

Definition at line 41 of file consdiffmgr.c.

◆ LABEL_VALID_UNTIL

#define LABEL_VALID_UNTIL "consensus-valid-until"

Definition at line 47 of file consdiffmgr.c.

◆ RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD

#define RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD ZLIB_METHOD

For which compression method do we retain old consensuses? There's no need to keep all of them, since we won't be serving them. We'll go with ZLIB_METHOD because it's pretty fast and everyone has it.

Definition at line 145 of file consdiffmgr.c.

Enumeration Type Documentation

◆ cdm_diff_status_t

enum cdm_diff_status_t

Possible status values for cdm_diff_t.cdm_diff_status

Definition at line 90 of file consdiffmgr.c.

Function Documentation

◆ cdm_cache_get()

STATIC consensus_cache_t * cdm_cache_get ( void )

Helper: return the consensus_cache_t * that backs this manager, initializing it if needed.

Definition at line 411 of file consdiffmgr.c.

Referenced by consdiffmgr_configure(), consdiffmgr_consensus_load(), consdiffmgr_diffs_load(), consdiffmgr_ensure_space_for_files(), consdiffmgr_register_with_sandbox(), consdiffmgr_rescan_flavor_(), consdiffmgr_set_cache_flags(), and consdiffmgr_validate().

◆ cdm_cache_init()

static void cdm_cache_init ( void )

static

Helper: initialize cons_diff_cache.

Definition at line 386 of file consdiffmgr.c.

◆ cdm_cache_lookup_consensus()

STATIC consensus_cache_entry_t * cdm_cache_lookup_consensus	(	consensus_flavor_t	flavor,
		time_t	valid_after
	)

Helper: look for a consensus with the given flavor and valid_after time in the cache. Return that consensus if it's present, or NULL if it's missing.

Definition at line 469 of file consdiffmgr.c.

◆ cdm_diff_eq()

static int cdm_diff_eq	(	const cdm_diff_t *	diff1,
		const cdm_diff_t *	diff2
	)

static

Helper: compare two cdm_diff_t objects for key equality

Definition at line 224 of file consdiffmgr.c.

◆ cdm_diff_free_()

static void cdm_diff_free_ ( cdm_diff_t * diff )

static

Release all storage held in diff.

Definition at line 240 of file consdiffmgr.c.

◆ cdm_diff_hash()

static unsigned cdm_diff_hash ( const cdm_diff_t * diff )

static

Helper: hash the key of a cdm_diff_t.

Definition at line 214 of file consdiffmgr.c.

◆ cdm_diff_ht_check_and_note_pending()

static int cdm_diff_ht_check_and_note_pending	(	consensus_flavor_t	flav,
		const uint8_t *	from_sha3,
		const uint8_t *	target_sha3
	)

static

Examine the diff hashtable to see whether we know anything about computing a diff of type flav between consensuses with the two provided SHA3-256 digests. If a computation is in progress, or if the computation has already been tried and failed, return 1. Otherwise, note the computation as "in progress" so that we don't reattempt it later, and return 0.

Definition at line 274 of file consdiffmgr.c.

◆ cdm_diff_ht_purge()

static void cdm_diff_ht_purge	(	consensus_flavor_t	flav,
		const uint8_t *	unless_target_sha3_matches
	)

static

Helper: Remove from the hash table every present (actually computed) diff of type flav whose target digest does not match unless_target_sha3_matches.

This function is used for the hash table to throw away references to diffs that do not lead to the most given consensus of a given flavor.

Definition at line 351 of file consdiffmgr.c.

◆ cdm_diff_ht_set_status()

static void cdm_diff_ht_set_status	(	consensus_flavor_t	flav,
		const uint8_t *	from_sha3,
		const uint8_t *	to_sha3,
		compress_method_t	method,
		int	status,
		consensus_cache_entry_handle_t *	handle
	)

static

Update the status of the diff of type flav between consensuses with the two provided SHA3-256 digests, so that its status becomes status, and its value becomes the handle. If handle is NULL, then the old handle (if any) is freed, and replaced with NULL.

Definition at line 307 of file consdiffmgr.c.

◆ cdm_diff_new()

static cdm_diff_t * cdm_diff_new	(	consensus_flavor_t	flav,
		const uint8_t *	from_sha3,
		const uint8_t *	target_sha3,
		compress_method_t	method
	)

static

Create and return a new cdm_diff_t with the given values. Does not add it to the hashtable.

Definition at line 251 of file consdiffmgr.c.

◆ cdm_entry_get_sha3_value()

STATIC int cdm_entry_get_sha3_value	(	uint8_t *	digest_out,
		consensus_cache_entry_t *	ent,
		const char *	label
	)

Helper: if there is a sha3-256 hex-encoded digest in ent with the given label, set digest_out to that value (decoded), and return 0.

Return -1 if there is no such label, and -2 if it is badly formatted.

Definition at line 445 of file consdiffmgr.c.

Referenced by consdiffmgr_validate().

◆ cdm_labels_prepend_sha3()

static void cdm_labels_prepend_sha3	(	config_line_t **	labels,
		const char *	label,
		const uint8_t *	body,
		size_t	bodylen
	)

static

Helper: given a list of labels, prepend the hex-encoded SHA3 digest of the bodylen-byte object at body to those labels, with label as its label.

Definition at line 425 of file consdiffmgr.c.

Referenced by consensus_compress_worker_threadfn().

◆ compare_by_staleness_()

static int compare_by_staleness_	(	const void **	a,
		const void **	b
	)

static

Helper: compare two files by their from-valid-after and valid-after labels, trying to sort in ascending order by from-valid-after (when present) and valid-after (when not). Place everything that has neither label first in the list.

Definition at line 1168 of file consdiffmgr.c.

Referenced by consdiffmgr_ensure_space_for_files().

◆ compare_by_valid_after_()

static int compare_by_valid_after_	(	const void **	a,
		const void **	b
	)

static

Helper: used to sort two smartlists of consensus_cache_entry_t by their LABEL_VALID_AFTER labels.

Definition at line 584 of file consdiffmgr.c.

Referenced by sort_and_find_most_recent().

◆ compress_multiple()

static int compress_multiple	(	compressed_result_t *	results_out,
		int	n_methods,
		const compress_method_t *	methods,
		const uint8_t *	input,
		size_t	len,
		const config_line_t *	labels_in
	)

static

Compress the bytestring input of length len using the n_methods compression methods listed in the array methods.

For each successful compression, set the fields in the results_out array in the position corresponding to the compression method. Use labels_in as a basis for the labels of the result.

Return 0 if all compression succeeded; -1 if any failed.

Definition at line 1315 of file consdiffmgr.c.

◆ consdiffmgr_add_consensus()

int consdiffmgr_add_consensus	(	const char *	consensus,
		size_t	consensus_len,
		const networkstatus_t *	as_parsed
	)

Given a buffer containing a networkstatus consensus, and the results of having parsed that consensus, add that consensus to the cache if it is not already present and not too old. Create new consensus diffs from or to that consensus as appropriate.

Return 0 on success and -1 on failure.

Definition at line 549 of file consdiffmgr.c.

◆ consdiffmgr_cleanup()

int consdiffmgr_cleanup ( void )

Perform periodic cleanup tasks on the consensus diff cache. Return the number of objects marked for deletion.

Definition at line 722 of file consdiffmgr.c.

Referenced by consdiffmgr_ensure_space_for_files(), and consdiffmgr_rescan().

◆ consdiffmgr_configure()

void consdiffmgr_configure ( const consdiff_cfg_t * cfg )

Initialize the consensus diff manager and its cache, and configure its parameters based on the latest torrc and networkstatus parameters.

Definition at line 844 of file consdiffmgr.c.

◆ consdiffmgr_consensus_load()

static void consdiffmgr_consensus_load ( void )

static

Scan the cache for the latest consensuses and add their handles to latest_consensus

Definition at line 1038 of file consdiffmgr.c.

Referenced by consdiffmgr_rescan().

◆ consdiffmgr_diffs_load()

static void consdiffmgr_diffs_load ( void )

static

Scan the cache for diffs, and add them to the hashtable.

Definition at line 1079 of file consdiffmgr.c.

Referenced by consdiffmgr_rescan().

◆ consdiffmgr_enable_background_compression()

void consdiffmgr_enable_background_compression ( void )

Tell the consdiffmgr backend to compress consensuses in worker threads.

Definition at line 1918 of file consdiffmgr.c.

◆ consdiffmgr_ensure_space_for_files()

static int consdiffmgr_ensure_space_for_files ( int n )

static

If there are not enough unused filenames to store n files, then delete old consensuses until there are. (We have to keep track of the number of filenames because of the way that the seccomp2 cache works.)

Return 0 on success, -1 on failure.

Definition at line 1195 of file consdiffmgr.c.

◆ consdiffmgr_find_consensus()

consdiff_status_t consdiffmgr_find_consensus	(	struct consensus_cache_entry_t **	entry_out,
		consensus_flavor_t	flavor,
		compress_method_t	method
	)

If we know a consensus with the flavor flavor compressed with method, set *entry_out to that value. Return values are as for consdiffmgr_find_diff_from().

Definition at line 629 of file consdiffmgr.c.

◆ consdiffmgr_find_diff_from()

consdiff_status_t consdiffmgr_find_diff_from	(	consensus_cache_entry_t **	entry_out,
		consensus_flavor_t	flavor,
		int	digest_type,
		const uint8_t *	digest,
		size_t	digestlen,
		compress_method_t	method
	)

Look up consensus_cache_entry_t for the consensus of type flavor, from the source consensus with the specified digest (which must be SHA3).

If the diff is present, store it into *entry_out and return CONSDIFF_AVAILABLE. Otherwise return CONSDIFF_NOT_FOUND or CONSDIFF_IN_PROGRESS.

Definition at line 660 of file consdiffmgr.c.

◆ consdiffmgr_free_all()

void consdiffmgr_free_all ( void )

Called before shutdown: drop all storage held by the consdiffmgr.c module.

Definition at line 1267 of file consdiffmgr.c.

◆ consdiffmgr_register_with_sandbox()

int consdiffmgr_register_with_sandbox ( struct sandbox_cfg_elem_t ** cfg )

Tell the sandbox (if any) configured by cfg to allow the operations that the consensus diff manager will need.

Definition at line 857 of file consdiffmgr.c.

◆ consdiffmgr_rescan()

void consdiffmgr_rescan ( void )

Build new diffs as needed.

Definition at line 1121 of file consdiffmgr.c.

Referenced by consdiffmgr_rescan_cb().

◆ consdiffmgr_rescan_cb()

static void consdiffmgr_rescan_cb	(	mainloop_event_t *	ev,
		void *	arg
	)

static

Callback wrapper for consdiffmgr_rescan

Definition at line 1145 of file consdiffmgr.c.

◆ consdiffmgr_rescan_flavor_()

static void consdiffmgr_rescan_flavor_ ( consensus_flavor_t flavor )

static

Helper: build new diffs of flavor as needed

Definition at line 920 of file consdiffmgr.c.

Referenced by consdiffmgr_rescan().

◆ consdiffmgr_set_cache_flags()

static void consdiffmgr_set_cache_flags ( void )

static

Set consensus cache flags on the objects in this consdiffmgr.

Definition at line 1250 of file consdiffmgr.c.

◆ consdiffmgr_validate()

int consdiffmgr_validate ( void )

Scan the consensus diff manager's cache for any grossly malformed entries, and mark them as deletable. Return 0 if no problems were found; 1 if problems were found and fixed.

Definition at line 868 of file consdiffmgr.c.

◆ consensus_cache_entry_get_fresh_until()

int consensus_cache_entry_get_fresh_until	(	const consensus_cache_entry_t *	ent,
		time_t *	out
	)

Read the fresh-until time of cached object ent into *out and return 0, or return -1 if no such time was recorded.

Definition at line 1944 of file consdiffmgr.c.

◆ consensus_cache_entry_get_valid_after()

int consensus_cache_entry_get_valid_after	(	const consensus_cache_entry_t *	ent,
		time_t *	out
	)

Read the valid after timestamp from the cached object ent into *out and return 0, or return -1 if no such time was recorded.

Definition at line 1977 of file consdiffmgr.c.

Referenced by spooled_resource_estimate_size().

◆ consensus_cache_entry_get_valid_until()

int consensus_cache_entry_get_valid_until	(	const consensus_cache_entry_t *	ent,
		time_t *	out
	)

Read the valid until timestamp from the cached object ent into *out and return 0, or return -1 if no such time was recorded.

Definition at line 1960 of file consdiffmgr.c.

◆ consensus_cache_entry_get_voter_id_digests()

int consensus_cache_entry_get_voter_id_digests	(	const consensus_cache_entry_t *	ent,
		smartlist_t *	out
	)

Read the set of voters from the cached object ent into out, as a list of hex-encoded digests. Return 0 on success, -1 if no signatories were recorded.

Definition at line 1928 of file consdiffmgr.c.

Referenced by client_likes_consensus().

◆ consensus_compress_worker_job_free_()

static void consensus_compress_worker_job_free_ ( consensus_compress_worker_job_t * job )

static

Free all resources held in job

Definition at line 1760 of file consdiffmgr.c.

◆ consensus_compress_worker_replyfn()

static void consensus_compress_worker_replyfn ( void * work_ )

static

Worker function: This function runs in the main thread, and receives a consensus_diff_compress_job_t that the worker thread has already processed.

Definition at line 1819 of file consdiffmgr.c.

◆ consensus_compress_worker_threadfn()

static workqueue_reply_t consensus_compress_worker_threadfn	(	void *	state_,
		void *	work_
	)

static

Worker function. This function runs inside a worker thread and receives a consensus_compress_worker_job_t as its input.

Definition at line 1778 of file consdiffmgr.c.

◆ consensus_compression_method_pos()

static int consensus_compression_method_pos ( compress_method_t method )

static

Return i such that compress_consensus_with[i] == method. Return -1 if no such i exists.

Definition at line 612 of file consdiffmgr.c.

◆ consensus_diff_queue_diff_work()

static int consensus_diff_queue_diff_work	(	consensus_cache_entry_t *	diff_from,
		consensus_cache_entry_t *	diff_to
	)

static

Queue the job of computing the diff from diff_from to diff_to in a worker thread.

Definition at line 1707 of file consdiffmgr.c.

◆ consensus_diff_worker_job_free_()

static void consensus_diff_worker_job_free_ ( consensus_diff_worker_job_t * job )

static

Helper: release all storage held in job.

Definition at line 1604 of file consdiffmgr.c.

◆ consensus_diff_worker_replyfn()

static void consensus_diff_worker_replyfn ( void * work_ )

static

Worker function: This function runs in the main thread, and receives a consensus_diff_worker_job_t that the worker thread has already processed.

Definition at line 1624 of file consdiffmgr.c.

◆ consensus_diff_worker_threadfn()

static workqueue_reply_t consensus_diff_worker_threadfn	(	void *	state_,
		void *	work_
	)

static

Worker function. This function runs inside a worker thread and receives a consensus_diff_worker_job_t as its input.

Definition at line 1462 of file consdiffmgr.c.

◆ consensus_queue_compression_work()

static int consensus_queue_compression_work	(	const char *	consensus,
		size_t	consensus_len,
		const networkstatus_t *	as_parsed
	)

static

Queue a job to compress consensus and store its compressed text in the cache.

Definition at line 1857 of file consdiffmgr.c.

◆ get_max_age_to_cache()

static int32_t get_max_age_to_cache ( void )

static

Return the maximum age (in seconds) of consensuses that we should consider storing. The available space in the directory may impose additional limits on how much we store.

Definition at line 498 of file consdiffmgr.c.

◆ HT_HEAD()

static HT_HEAD	(	cdm_diff_ht	,
		cdm_diff_t
	)

static

Hashtable mapping flavor and source consensus digest to status. Configuration for this module

Definition at line 178 of file consdiffmgr.c.

◆ mark_cdm_cache_dirty()

static void mark_cdm_cache_dirty ( void )

static

Mark the cache as dirty, and schedule a rescan event.

Definition at line 1154 of file consdiffmgr.c.

Referenced by consensus_compress_worker_replyfn().

◆ n_consensus_compression_methods()

STATIC unsigned n_consensus_compression_methods ( void )

How many different methods will we try to use for diff compression?

Definition at line 136 of file consdiffmgr.c.

Referenced by consensus_compress_worker_replyfn().

◆ n_diff_compression_methods()

STATIC unsigned n_diff_compression_methods ( void )

How many different methods will we try to use for diff compression?

Definition at line 118 of file consdiffmgr.c.

Referenced by cdm_diff_ht_check_and_note_pending(), and consensus_diff_worker_job_free_().

◆ sort_and_find_most_recent()

static consensus_cache_entry_t * sort_and_find_most_recent ( smartlist_t * lst )

static

Helper: Sort lst by LABEL_VALID_AFTER and return the most recent entry.

Definition at line 599 of file consdiffmgr.c.

◆ store_multiple()

static cdm_diff_status_t store_multiple	(	consensus_cache_entry_handle_t **	handles_out,
		int	n,
		const compress_method_t *	methods,
		const compressed_result_t *	results,
		const char *	description
	)

static

Given an array of n compressed_result_t in results, as produced by compress_multiple, store them all into the consdiffmgr, and store handles to them in the handles_out array.

Return CDM_DIFF_PRESENT if any was stored, and CDM_DIFF_ERROR if none was stored.

Definition at line 1354 of file consdiffmgr.c.

Referenced by consensus_compress_worker_replyfn().

◆ uncompress_or_set_ptr()

STATIC int uncompress_or_set_ptr	(	const char **	out,
		size_t *	outlen,
		char **	owned_out,
		consensus_cache_entry_t *	ent
	)

Given a consensus_cache_entry_t, check whether it has a label claiming that it was compressed. If so, uncompress its contents into *out and set outlen to hold their size, and set *owned_out to a pointer that the caller will need to free. If not, just set *out and outlen to its extent in memory. Return 0 on success, -1 on failure.

Definition at line 1425 of file consdiffmgr.c.

Variable Documentation

◆ background_compression

int background_compression = 0

static

If true, we compress in worker threads.

Definition at line 1850 of file consdiffmgr.c.

Referenced by consdiffmgr_enable_background_compression().

◆ cdm_cache_dirty

int cdm_cache_dirty = 0

static

If true, we have learned at least one new consensus since the consensus cache was last up-to-date.

Definition at line 81 of file consdiffmgr.c.

Referenced by consdiffmgr_rescan(), and mark_cdm_cache_dirty().

◆ cdm_cache_loaded

int cdm_cache_loaded = 0

static

If true, we have scanned the cache to update our hashtable of diffs.

Definition at line 85 of file consdiffmgr.c.

Referenced by consdiffmgr_rescan().

◆ compress_consensus_with

const compress_method_t compress_consensus_with[]

static

Initial value:

= {
  ZLIB_METHOD,
 
}

Which methods do we use for precompressing consensuses?

Definition at line 124 of file consdiffmgr.c.

Referenced by consensus_compress_worker_replyfn(), and n_consensus_compression_methods().

◆ compress_diffs_with

const compress_method_t compress_diffs_with[]

static

Initial value:

= {
  NO_METHOD,
  GZIP_METHOD,
 
}

Which methods do we use for precompressing diffs?

Definition at line 97 of file consdiffmgr.c.

Referenced by n_diff_compression_methods().

◆ cons_diff_cache

consensus_cache_t* cons_diff_cache = NULL

static

Underlying directory that stores consensuses and consensus diffs. Don't use this directly: use cdm_cache_get() instead.

Definition at line 76 of file consdiffmgr.c.

Referenced by cdm_cache_init().

◆ consdiffmgr_rescan_ev

mainloop_event_t* consdiffmgr_rescan_ev = NULL

static

Event for rescanning the cache.

Definition at line 111 of file consdiffmgr.c.

Referenced by mark_cdm_cache_dirty().

◆ latest_consensus

consensus_cache_entry_handle_t* latest_consensus[N_CONSENSUS_FLAVORS][ARRAY_LENGTH(compress_consensus_with)]

static

Handles pointing to the latest consensus entries as compressed and stored.

Definition at line 150 of file consdiffmgr.c.

Data Structures

Macros

Enumerations

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ cdm_diff_free

◆ CONSDIFFMGR_PRIVATE

◆ consensus_compress_worker_job_free

◆ consensus_diff_worker_job_free

◆ DOCTYPE_CONSENSUS

◆ DOCTYPE_CONSENSUS_DIFF

◆ LABEL_COMPRESSION_TYPE

◆ LABEL_DOCTYPE

◆ LABEL_FLAVOR

◆ LABEL_FRESH_UNTIL

◆ LABEL_FROM_SHA3_DIGEST

◆ LABEL_FROM_VALID_AFTER

◆ LABEL_SHA3_DIGEST

◆ LABEL_SHA3_DIGEST_AS_SIGNED

◆ LABEL_SHA3_DIGEST_UNCOMPRESSED

◆ LABEL_SIGNATORIES

◆ LABEL_TARGET_SHA3_DIGEST

◆ LABEL_VALID_AFTER

◆ LABEL_VALID_UNTIL

◆ RETAIN_CONSENSUS_COMPRESSED_WITH_METHOD

Enumeration Type Documentation

◆ cdm_diff_status_t

Function Documentation

◆ cdm_cache_get()

◆ cdm_cache_init()

◆ cdm_cache_lookup_consensus()

◆ cdm_diff_eq()

◆ cdm_diff_free_()

◆ cdm_diff_hash()

◆ cdm_diff_ht_check_and_note_pending()

◆ cdm_diff_ht_purge()

◆ cdm_diff_ht_set_status()

◆ cdm_diff_new()

◆ cdm_entry_get_sha3_value()

◆ cdm_labels_prepend_sha3()

◆ compare_by_staleness_()

◆ compare_by_valid_after_()

◆ compress_multiple()

◆ consdiffmgr_add_consensus()

◆ consdiffmgr_cleanup()

◆ consdiffmgr_configure()

◆ consdiffmgr_consensus_load()

◆ consdiffmgr_diffs_load()

◆ consdiffmgr_enable_background_compression()

◆ consdiffmgr_ensure_space_for_files()

◆ consdiffmgr_find_consensus()

◆ consdiffmgr_find_diff_from()

◆ consdiffmgr_free_all()

◆ consdiffmgr_register_with_sandbox()

◆ consdiffmgr_rescan()

◆ consdiffmgr_rescan_cb()

◆ consdiffmgr_rescan_flavor_()

◆ consdiffmgr_set_cache_flags()

◆ consdiffmgr_validate()

◆ consensus_cache_entry_get_fresh_until()

◆ consensus_cache_entry_get_valid_after()

◆ consensus_cache_entry_get_valid_until()

◆ consensus_cache_entry_get_voter_id_digests()

◆ consensus_compress_worker_job_free_()

◆ consensus_compress_worker_replyfn()

◆ consensus_compress_worker_threadfn()

◆ consensus_compression_method_pos()

◆ consensus_diff_queue_diff_work()

◆ consensus_diff_worker_job_free_()

◆ consensus_diff_worker_replyfn()

◆ consensus_diff_worker_threadfn()

◆ consensus_queue_compression_work()

◆ get_max_age_to_cache()

◆ HT_HEAD()

◆ mark_cdm_cache_dirty()

◆ n_consensus_compression_methods()

◆ n_diff_compression_methods()

◆ sort_and_find_most_recent()