Lines
91.15 %
Functions
62.22 %
Branches
100 %
//! Declare a general purpose "document ID type" for tracking which
//! documents we want and which we have.
use std::collections::HashMap;
use tracing::trace;
use crate::storage::Store;
use crate::DocumentText;
use tor_dirclient::request;
#[cfg(feature = "routerdesc")]
use tor_netdoc::doc::routerdesc::RdDigest;
use tor_netdoc::doc::{authcert::AuthCertKeyIds, microdesc::MdDigest, netstatus::ConsensusFlavor};
/// The identity of a single document, in enough detail to load it
/// from storage.
#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
#[non_exhaustive]
pub enum DocId {
/// A request for the most recent consensus document.
LatestConsensus {
/// The flavor of consensus to request.
flavor: ConsensusFlavor,
/// Rules for loading this consensus from the cache.
cache_usage: CacheUsage,
},
/// A request for an authority certificate, by the SHA1 digests of
/// its identity key and signing key.
AuthCert(AuthCertKeyIds),
/// A request for a single microdescriptor, by SHA256 digest.
Microdesc(MdDigest),
/// A request for the router descriptor of a public relay, by SHA1
/// digest.
RouterDesc(RdDigest),
}
/// The underlying type of a DocId.
///
/// Documents with the same type can be grouped into the same query; others
/// cannot.
#[derive(Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
pub(crate) enum DocType {
/// A consensus document
Consensus(ConsensusFlavor),
/// An authority certificate
AuthCert,
/// A microdescriptor
Microdesc,
/// A router descriptor.
RouterDesc,
impl DocId {
/// Return the associated doctype of this DocId.
pub(crate) fn doctype(&self) -> DocType {
use DocId::*;
use DocType as T;
match self {
LatestConsensus { flavor: f, .. } => T::Consensus(*f),
AuthCert(_) => T::AuthCert,
Microdesc(_) => T::Microdesc,
RouterDesc(_) => T::RouterDesc,
/// A request for a specific kind of directory resource that a DirMgr can
/// request.
#[derive(Clone, Debug)]
pub(crate) enum ClientRequest {
/// Request for a consensus
Consensus(request::ConsensusRequest),
/// Request for one or more authority certificates
AuthCert(request::AuthCertRequest),
/// Request for one or more microdescriptors
Microdescs(request::MicrodescRequest),
/// Request for one or more router descriptors
RouterDescs(request::RouterDescRequest),
impl ClientRequest {
/// Turn a ClientRequest into a Requestable.
pub(crate) fn as_requestable(&self) -> &(dyn request::Requestable + Send + Sync) {
use ClientRequest::*;
Consensus(a) => a,
AuthCert(a) => a,
Microdescs(a) => a,
RouterDescs(a) => a,
/// Description of how to start out a given bootstrap attempt.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
pub enum CacheUsage {
/// The bootstrap attempt will only use the cache. Therefore, don't
/// load a pending consensus from the cache, since we won't be able
/// to find enough information to make it usable.
CacheOnly,
/// The bootstrap attempt is willing to download information or to
/// use the cache. Therefore, we want the latest cached
/// consensus, whether it is pending or not.
CacheOkay,
/// The bootstrap attempt is trying to fetch a new consensus. Therefore,
/// we don't want a consensus from the cache.
MustDownload,
impl CacheUsage {
/// Turn this CacheUsage into a pending field for use with
/// SqliteStorage.
pub(crate) fn pending_requirement(&self) -> Option<bool> {
CacheUsage::CacheOnly => Some(false),
_ => None,
/// A group of DocIds that can be downloaded or loaded from the database
/// together.
/// TODO: Perhaps this should be the same as ClientRequest?
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum DocQuery {
/// A request for the latest consensus
/// A desired flavor of consensus
/// Whether we can or must use the cache
/// A request for authority certificates
AuthCert(Vec<AuthCertKeyIds>),
/// A request for microdescriptors
Microdesc(Vec<MdDigest>),
/// A request for router descriptors
RouterDesc(Vec<RdDigest>),
impl DocQuery {
/// Construct an "empty" docquery from the given DocId
pub(crate) fn empty_from_docid(id: &DocId) -> Self {
match *id {
DocId::LatestConsensus {
flavor,
cache_usage,
} => Self::LatestConsensus {
DocId::AuthCert(_) => Self::AuthCert(Vec::new()),
DocId::Microdesc(_) => Self::Microdesc(Vec::new()),
DocId::RouterDesc(_) => Self::RouterDesc(Vec::new()),
/// Add `id` to this query, if possible.
fn push(&mut self, id: DocId) {
match (self, id) {
(Self::LatestConsensus { .. }, DocId::LatestConsensus { .. }) => {}
(Self::AuthCert(ids), DocId::AuthCert(id)) => ids.push(id),
(Self::Microdesc(ids), DocId::Microdesc(id)) => ids.push(id),
(Self::RouterDesc(ids), DocId::RouterDesc(id)) => ids.push(id),
(_, _) => panic!(),
/// If this query contains too many documents to download with a single
/// request, divide it up.
pub(crate) fn split_for_download(self) -> Vec<Self> {
use DocQuery::*;
/// How many objects can be put in a single HTTP GET line?
const N: usize = 500;
LatestConsensus { .. } => vec![self],
AuthCert(mut v) => {
v.sort_unstable();
v[..].chunks(N).map(|s| AuthCert(s.to_vec())).collect()
Microdesc(mut v) => {
v[..].chunks(N).map(|s| Microdesc(s.to_vec())).collect()
RouterDesc(mut v) => {
v[..].chunks(N).map(|s| RouterDesc(s.to_vec())).collect()
/// Load documents specified by this `DocQuery` from the store, if they can be found.
/// # Note
/// This function may not return all documents that the query asked for. If this happens, no
/// error will be returned. It is the caller's responsibility to handle this case.
pub(crate) fn load_from_store_into(
&self,
result: &mut HashMap<DocId, DocumentText>,
store: &dyn Store,
) -> crate::Result<()> {
} => {
if *cache_usage == CacheUsage::MustDownload {
// Do nothing: we don't want a cached consensus.
trace!("MustDownload is set; not checking for cached consensus.");
} else if let Some(c) =
store.latest_consensus(*flavor, cache_usage.pending_requirement())?
{
trace!("Found a reasonable consensus in the cache");
let id = DocId::LatestConsensus {
flavor: *flavor,
cache_usage: *cache_usage,
};
result.insert(id, c.into());
AuthCert(ids) => result.extend(
store
.authcerts(ids)?
.into_iter()
.map(|(id, c)| (DocId::AuthCert(id), DocumentText::from_string(c))),
),
Microdesc(digests) => {
result.extend(
.microdescs(digests)?
.map(|(id, md)| (DocId::Microdesc(id), DocumentText::from_string(md))),
);
RouterDesc(digests) => result.extend(
.routerdescs(digests)?
.map(|(id, rd)| (DocId::RouterDesc(id), DocumentText::from_string(rd))),
Ok(())
impl From<DocId> for DocQuery {
fn from(d: DocId) -> DocQuery {
let mut result = DocQuery::empty_from_docid(&d);
result.push(d);
result
/// Given a list of DocId, split them up into queries, by type.
pub(crate) fn partition_by_type<T>(collection: T) -> HashMap<DocType, DocQuery>
where
T: IntoIterator<Item = DocId>,
let mut result = HashMap::new();
for item in collection.into_iter() {
let tp = item.doctype();
.entry(tp)
.or_insert_with(|| DocQuery::empty_from_docid(&item))
.push(item);
#[cfg(test)]
mod test {
// @@ begin test lint list maintained by maint/add_warning @@
#![allow(clippy::bool_assert_comparison)]
#![allow(clippy::clone_on_copy)]
#![allow(clippy::dbg_macro)]
#![allow(clippy::mixed_attributes_style)]
#![allow(clippy::print_stderr)]
#![allow(clippy::print_stdout)]
#![allow(clippy::single_char_pattern)]
#![allow(clippy::unwrap_used)]
#![allow(clippy::unchecked_duration_subtraction)]
#![allow(clippy::useless_vec)]
#![allow(clippy::needless_pass_by_value)]
//! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
use super::*;
use tor_basic_utils::test_rng::testing_rng;
#[test]
fn doctype() {
assert_eq!(
flavor: ConsensusFlavor::Microdesc,
cache_usage: CacheUsage::CacheOkay,
.doctype(),
DocType::Consensus(ConsensusFlavor::Microdesc)
let auth_id = AuthCertKeyIds {
id_fingerprint: [10; 20].into(),
sk_fingerprint: [12; 20].into(),
assert_eq!(DocId::AuthCert(auth_id).doctype(), DocType::AuthCert);
assert_eq!(DocId::Microdesc([22; 32]).doctype(), DocType::Microdesc);
assert_eq!(DocId::RouterDesc([42; 20]).doctype(), DocType::RouterDesc);
fn partition_ids() {
let mut ids = Vec::new();
for byte in 0..=255 {
ids.push(DocId::Microdesc([byte; 32]));
ids.push(DocId::RouterDesc([byte; 20]));
ids.push(DocId::AuthCert(AuthCertKeyIds {
id_fingerprint: [byte; 20].into(),
sk_fingerprint: [33; 20].into(),
}));
let consensus_q = DocId::LatestConsensus {
ids.push(consensus_q);
let split = partition_by_type(ids);
assert_eq!(split.len(), 4); // 4 distinct types.
#[cfg(not(feature = "routerdesc"))]
assert_eq!(split.len(), 3); // 3 distinct types.
let q = split
.get(&DocType::Consensus(ConsensusFlavor::Microdesc))
.unwrap();
assert!(matches!(q, DocQuery::LatestConsensus { .. }));
let q = split.get(&DocType::Microdesc).unwrap();
assert!(matches!(q, DocQuery::Microdesc(v) if v.len() == 256));
let q = split.get(&DocType::RouterDesc).unwrap();
assert!(matches!(q, DocQuery::RouterDesc(v) if v.len() == 256));
let q = split.get(&DocType::AuthCert).unwrap();
assert!(matches!(q, DocQuery::AuthCert(v) if v.len() == 256));
fn split_into_chunks() {
use std::collections::HashSet;
//use itertools::Itertools;
use rand::Rng;
// Construct a big query.
let mut rng = testing_rng();
let ids: HashSet<MdDigest> = (0..3400).map(|_| rng.random()).collect();
// Test microdescs.
let split = DocQuery::Microdesc(ids.clone().into_iter().collect()).split_for_download();
assert_eq!(split.len(), 7);
let mut found_ids = HashSet::new();
for q in split {
match q {
DocQuery::Microdesc(ids) => ids.into_iter().for_each(|id| {
found_ids.insert(id);
}),
_ => panic!("Wrong type."),
assert_eq!(found_ids.len(), 3400);
assert_eq!(found_ids, ids);
// Test routerdescs.
let ids: HashSet<RdDigest> = (0..1001).map(|_| rng.random()).collect();
let split =
DocQuery::RouterDesc(ids.clone().into_iter().collect()).split_for_download();
assert_eq!(split.len(), 3);
DocQuery::RouterDesc(ids) => ids.into_iter().for_each(|id| {
assert_eq!(found_ids.len(), 1001);
assert_eq!(&found_ids, &ids);
// Test authcerts.
let ids: HashSet<AuthCertKeyIds> = (0..2500)
.map(|_| {
let id_fingerprint = rng.random::<[u8; 20]>().into();
let sk_fingerprint = rng.random::<[u8; 20]>().into();
AuthCertKeyIds {
id_fingerprint,
sk_fingerprint,
})
.collect();
let split = DocQuery::AuthCert(ids.clone().into_iter().collect()).split_for_download();
assert_eq!(split.len(), 5);
DocQuery::AuthCert(ids) => ids.into_iter().for_each(|id| {
assert_eq!(found_ids.len(), 2500);
// Consensus is trivial?
let query = DocQuery::LatestConsensus {
let split = query.clone().split_for_download();
assert_eq!(split, vec![query]);
fn into_query() {
let q: DocQuery = DocId::Microdesc([99; 32]).into();
assert_eq!(q, DocQuery::Microdesc(vec![[99; 32]]));
fn pending_requirement() {
// If we want to keep all of our activity within the cache,
// we must request a non-pending consensus from the cache.
assert_eq!(CacheUsage::CacheOnly.pending_requirement(), Some(false));
// Otherwise, any cached consensus, pending or not, will meet
// our needs.
assert_eq!(CacheUsage::CacheOkay.pending_requirement(), None);
assert_eq!(CacheUsage::MustDownload.pending_requirement(), None);