Lines
91.28 %
Functions
81.82 %
Branches
100 %
//! Parsing implementation for Tor microdescriptors.
//!
//! A "microdescriptor" is an incomplete, infrequently-changing
//! summary of a relay's information that is generated by
//! the directory authorities.
//! Microdescriptors are much smaller than router descriptors, and
//! change less frequently. For this reason, they're currently used
//! for building circuits by all relays and clients.
//! Microdescriptors can't be used on their own: you need to know
//! which relay they are for, which requires a valid consensus
//! directory.
use crate::parse::keyword::Keyword;
use crate::parse::parser::SectionRules;
use crate::parse::tokenize::{ItemResult, NetDocReader};
use crate::types::family::{RelayFamily, RelayFamilyId};
use crate::types::misc::*;
use crate::types::policy::PortPolicy;
use crate::util;
use crate::util::str::Extent;
use crate::util::PeekableIterator;
use crate::{AllowAnnotations, Error, NetdocErrorKind as EK, Result};
use tor_error::internal;
use tor_llcrypto::d;
use tor_llcrypto::pk::{curve25519, ed25519, rsa};
use digest::Digest;
use once_cell::sync::Lazy;
use std::str::FromStr as _;
use std::sync::Arc;
use std::time;
#[cfg(feature = "build_docs")]
mod build;
pub use build::MicrodescBuilder;
/// Annotations prepended to a microdescriptor that has been stored to
/// disk.
#[allow(dead_code)]
#[derive(Clone, Debug, Default)]
pub struct MicrodescAnnotation {
/// A time at which this microdescriptor was last listed in some
/// consensus document.
last_listed: Option<time::SystemTime>,
}
/// The digest of a microdescriptor as used in microdesc consensuses
pub type MdDigest = [u8; 32];
/// A single microdescriptor.
#[cfg_attr(
feature = "dangerous-expose-struct-fields",
visible::StructFields(pub),
non_exhaustive
)]
#[derive(Clone, Debug)]
pub struct Microdesc {
/// The SHA256 digest of the text of this microdescriptor. This
/// value is used to identify the microdescriptor when downloading
/// it, and when listing it in a consensus document.
// TODO: maybe this belongs somewhere else. Once it's used to store
// correlate the microdesc to a consensus, it's never used again.
#[cfg_attr(docsrs, doc(cfg(feature = "dangerous-expose-struct-fields")))]
sha256: MdDigest,
/// Public key used for the ntor circuit extension protocol.
ntor_onion_key: curve25519::PublicKey,
/// Declared family for this relay.
family: Arc<RelayFamily>,
/// List of IPv4 ports to which this relay will exit
ipv4_policy: Arc<PortPolicy>,
/// List of IPv6 ports to which this relay will exit
ipv6_policy: Arc<PortPolicy>,
/// Ed25519 identity for this relay
ed25519_id: ed25519::Ed25519Identity,
/// Family identities for this relay.
family_ids: Vec<RelayFamilyId>,
// addr is obsolete and doesn't go here any more
// pr is obsolete and doesn't go here any more.
// The legacy "tap" onion-key is obsolete, and though we parse it, we don't
// save it.
impl Microdesc {
/// Create a new MicrodescBuilder that can be used to construct
/// microdescriptors.
///
/// This function is only available when the crate is built with the
/// `build_docs` feature.
/// # Limitations
/// The generated microdescriptors cannot yet be encoded, and do
/// not yet have correct sha256 digests. As such they are only
/// useful for testing.
pub fn builder() -> MicrodescBuilder {
MicrodescBuilder::new()
/// Return the sha256 digest of this microdesc.
pub fn digest(&self) -> &MdDigest {
&self.sha256
/// Return the ntor onion key for this microdesc
pub fn ntor_key(&self) -> &curve25519::PublicKey {
&self.ntor_onion_key
/// Return the ipv4 exit policy for this microdesc
pub fn ipv4_policy(&self) -> &Arc<PortPolicy> {
&self.ipv4_policy
/// Return the ipv6 exit policy for this microdesc
pub fn ipv6_policy(&self) -> &Arc<PortPolicy> {
&self.ipv6_policy
/// Return the relay family for this microdesc
pub fn family(&self) -> &RelayFamily {
self.family.as_ref()
/// Return the ed25519 identity for this microdesc, if its
/// Ed25519 identity is well-formed.
pub fn ed25519_id(&self) -> &ed25519::Ed25519Identity {
&self.ed25519_id
/// Return a list of family ids for this microdesc.
pub fn family_ids(&self) -> &[RelayFamilyId] {
&self.family_ids[..]
/// A microdescriptor annotated with additional data
/// TODO: rename this.
pub struct AnnotatedMicrodesc {
/// The microdescriptor
md: Microdesc,
/// The annotations for the microdescriptor
ann: MicrodescAnnotation,
/// Where did we find the microdescriptor with the originally parsed
/// string?
location: Option<Extent>,
impl AnnotatedMicrodesc {
/// Consume this annotated microdesc and discard its annotations.
pub fn into_microdesc(self) -> Microdesc {
self.md
/// Return a reference to the microdescriptor within this annotated
/// microdescriptor.
pub fn md(&self) -> &Microdesc {
&self.md
/// If this Microdesc was parsed from `s`, return its original text.
pub fn within<'a>(&self, s: &'a str) -> Option<&'a str> {
self.location.as_ref().and_then(|ext| ext.reconstruct(s))
decl_keyword! {
/// Keyword type for recognized objects in microdescriptors.
MicrodescKwd {
annotation "@last-listed" => ANN_LAST_LISTED,
"onion-key" => ONION_KEY,
"ntor-onion-key" => NTOR_ONION_KEY,
"family" => FAMILY,
"family-ids" => FAMILY_IDS,
"p" => P,
"p6" => P6,
"id" => ID,
/// Rules about annotations that can appear before a Microdescriptor
static MICRODESC_ANNOTATIONS: Lazy<SectionRules<MicrodescKwd>> = Lazy::new(|| {
use MicrodescKwd::*;
let mut rules = SectionRules::builder();
rules.add(ANN_LAST_LISTED.rule().args(1..));
rules.add(ANN_UNRECOGNIZED.rule().may_repeat().obj_optional());
// unrecognized annotations are okay; anything else is a bug in this
// context.
rules.reject_unrecognized();
rules.build()
});
/// Rules about entries that must appear in an Microdesc, and how they must
/// be formed.
static MICRODESC_RULES: Lazy<SectionRules<MicrodescKwd>> = Lazy::new(|| {
rules.add(ONION_KEY.rule().required().no_args().obj_optional());
rules.add(NTOR_ONION_KEY.rule().required().args(1..));
rules.add(FAMILY.rule().args(1..));
rules.add(FAMILY_IDS.rule().args(0..));
rules.add(P.rule().args(2..));
rules.add(P6.rule().args(2..));
rules.add(ID.rule().may_repeat().args(2..));
rules.add(UNRECOGNIZED.rule().may_repeat().obj_optional());
impl MicrodescAnnotation {
/// Extract a (possibly empty) microdescriptor annotation from a
/// reader.
fn parse_from_reader(
reader: &mut NetDocReader<'_, MicrodescKwd>,
) -> Result<MicrodescAnnotation> {
let mut items = reader.pause_at(|item| item.is_ok_with_non_annotation());
let body = MICRODESC_ANNOTATIONS.parse(&mut items)?;
let last_listed = match body.get(ANN_LAST_LISTED) {
None => None,
Some(item) => Some(item.args_as_str().parse::<Iso8601TimeSp>()?.into()),
};
Ok(MicrodescAnnotation { last_listed })
/// Parse a string into a new microdescriptor.
pub fn parse(s: &str) -> Result<Microdesc> {
let mut items = crate::parse::tokenize::NetDocReader::new(s)?;
let (result, _) = Self::parse_from_reader(&mut items).map_err(|e| e.within(s))?;
items.should_be_exhausted()?;
Ok(result)
/// Extract a single microdescriptor from a NetDocReader.
) -> Result<(Microdesc, Option<Extent>)> {
let s = reader.str();
let mut first_onion_key = true;
// We'll pause at the next annotation, or at the _second_ onion key.
let mut items = reader.pause_at(|item| match item {
Err(_) => false,
Ok(item) => {
item.kwd().is_annotation()
|| if item.kwd() == ONION_KEY {
let was_first = first_onion_key;
first_onion_key = false;
!was_first
} else {
false
let body = MICRODESC_RULES.parse(&mut items)?;
// We have to start with onion-key
let start_pos = {
// unwrap here is safe because parsing would have failed
// had there not been at least one item.
#[allow(clippy::unwrap_used)]
let first = body.first_item().unwrap();
if first.kwd() != ONION_KEY {
return Err(EK::WrongStartingToken
.with_msg(first.kwd_str().to_string())
.at_pos(first.pos()));
// Unwrap is safe here because we are parsing these strings from s
util::str::str_offset(s, first.kwd_str()).unwrap()
// Legacy (tap) onion key. We parse this to make sure it's well-formed,
// but then we discard it immediately, since we never want to use it.
//
// In microdescriptors, the ONION_KEY field is mandatory, but its
// associated object is optional.
{
let tok = body.required(ONION_KEY)?;
if tok.has_obj() {
let _: rsa::PublicKey = tok
.parse_obj::<RsaPublic>("RSA PUBLIC KEY")?
.check_len_eq(1024)?
.check_exponent(65537)?
.into();
// Ntor onion key
let ntor_onion_key = body
.required(NTOR_ONION_KEY)?
.parse_arg::<Curve25519Public>(0)?
// family
// (We don't need to add the relay's own ID to this family, as we do in
// RouterDescs: the authorities already took care of that for us.)
let family = body
.maybe(FAMILY)
.parse_args_as_str::<RelayFamily>()?
.unwrap_or_else(RelayFamily::new)
.intern();
// Family ids (happy families case).
let family_ids = body
.maybe(FAMILY_IDS)
.args_as_str()
.unwrap_or("")
.split_ascii_whitespace()
.map(RelayFamilyId::from_str)
.collect::<Result<_>>()?;
// exit policies.
let ipv4_policy = body
.maybe(P)
.parse_args_as_str::<PortPolicy>()?
.unwrap_or_else(PortPolicy::new_reject_all);
let ipv6_policy = body
.maybe(P6)
// ed25519 identity
let ed25519_id = {
let id_tok = body
.slice(ID)
.iter()
.find(|item| item.arg(0) == Some("ed25519"));
match id_tok {
None => {
return Err(EK::MissingToken.with_msg("id ed25519"));
Some(tok) => tok.parse_arg::<Ed25519Public>(1)?.into(),
let end_pos = {
let last_item = body.last_item().unwrap();
last_item.offset_after(s).ok_or_else(|| {
Error::from(internal!("last item was not within source string"))
.at_pos(last_item.end_pos())
})?
let text = &s[start_pos..end_pos];
let sha256 = d::Sha256::digest(text.as_bytes()).into();
let location = Extent::new(s, text);
let md = Microdesc {
sha256,
ntor_onion_key,
family,
ipv4_policy: ipv4_policy.intern(),
ipv6_policy: ipv6_policy.intern(),
ed25519_id,
family_ids,
Ok((md, location))
/// Consume tokens from 'reader' until the next token is the beginning
/// of a microdescriptor: an annotation or an ONION_KEY. If no such
/// token exists, advance to the end of the reader.
fn advance_to_next_microdesc(reader: &mut NetDocReader<'_, MicrodescKwd>, annotated: bool) {
loop {
let item = reader.peek();
match item {
Some(Ok(t)) => {
let kwd = t.kwd();
if (annotated && kwd.is_annotation()) || kwd == ONION_KEY {
return;
Some(Err(_)) => {
// We skip over broken tokens here.
// (This case can't happen in practice, since if there had been
// any error tokens, they would have been handled as part of
// handling the previous microdesc.)
let _ = reader.next();
/// An iterator that parses one or more (possibly annotated)
/// microdescriptors from a string.
#[derive(Debug)]
pub struct MicrodescReader<'a> {
/// True if we accept annotations; false otherwise.
annotated: bool,
/// An underlying reader to give us Items for the microdescriptors
reader: NetDocReader<'a, MicrodescKwd>,
impl<'a> MicrodescReader<'a> {
/// Construct a MicrodescReader to take microdescriptors from a string
/// 's'.
pub fn new(s: &'a str, allow: &AllowAnnotations) -> Result<Self> {
let reader = NetDocReader::new(s)?;
let annotated = allow == &AllowAnnotations::AnnotationsAllowed;
Ok(MicrodescReader { annotated, reader })
/// If we're annotated, parse an annotation from the reader. Otherwise
/// return a default annotation.
fn take_annotation(&mut self) -> Result<MicrodescAnnotation> {
if self.annotated {
MicrodescAnnotation::parse_from_reader(&mut self.reader)
Ok(MicrodescAnnotation::default())
/// Parse a (possibly annotated) microdescriptor from the reader.
/// On error, parsing stops after the first failure.
fn take_annotated_microdesc_raw(&mut self) -> Result<AnnotatedMicrodesc> {
let ann = self.take_annotation()?;
let (md, location) = Microdesc::parse_from_reader(&mut self.reader)?;
Ok(AnnotatedMicrodesc { md, ann, location })
/// On error, advance the reader to the start of the next microdescriptor.
fn take_annotated_microdesc(&mut self) -> Result<AnnotatedMicrodesc> {
let pos_orig = self.reader.pos();
let result = self.take_annotated_microdesc_raw();
if result.is_err() {
if self.reader.pos() == pos_orig {
// No tokens were consumed from the reader. We need to
// drop at least one token to ensure we aren't looping.
// (This might not be able to happen, but it's easier to
// explicitly catch this case than it is to prove that
// it's impossible.)
let _ = self.reader.next();
advance_to_next_microdesc(&mut self.reader, self.annotated);
result
impl<'a> Iterator for MicrodescReader<'a> {
type Item = Result<AnnotatedMicrodesc>;
fn next(&mut self) -> Option<Self::Item> {
// If there is no next token, we're at the end.
self.reader.peek()?;
Some(
self.take_annotated_microdesc()
.map_err(|e| e.within(self.reader.str())),
)
#[cfg(test)]
mod test {
// @@ begin test lint list maintained by maint/add_warning @@
#![allow(clippy::bool_assert_comparison)]
#![allow(clippy::clone_on_copy)]
#![allow(clippy::dbg_macro)]
#![allow(clippy::mixed_attributes_style)]
#![allow(clippy::print_stderr)]
#![allow(clippy::print_stdout)]
#![allow(clippy::single_char_pattern)]
#![allow(clippy::unwrap_used)]
#![allow(clippy::unchecked_duration_subtraction)]
#![allow(clippy::useless_vec)]
#![allow(clippy::needless_pass_by_value)]
//! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
use super::*;
use hex_literal::hex;
const TESTDATA: &str = include_str!("../../testdata/microdesc1.txt");
const TESTDATA2: &str = include_str!("../../testdata/microdesc2.txt");
const TESTDATA3: &str = include_str!("../../testdata/microdesc3.txt");
const TESTDATA4: &str = include_str!("../../testdata/microdesc4.txt");
fn read_bad(fname: &str) -> String {
use std::fs;
use std::path::PathBuf;
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.push("testdata");
path.push("bad-mds");
path.push(fname);
fs::read_to_string(path).unwrap()
#[test]
fn parse_single() -> Result<()> {
let _md = Microdesc::parse(TESTDATA)?;
Ok(())
fn parse_no_tap_key() -> Result<()> {
let _md = Microdesc::parse(TESTDATA3)?;
fn parse_multi() -> Result<()> {
use humantime::parse_rfc3339;
let mds: Result<Vec<_>> =
MicrodescReader::new(TESTDATA2, &AllowAnnotations::AnnotationsAllowed)?.collect();
let mds = mds?;
assert_eq!(mds.len(), 4);
assert_eq!(
mds[0].ann.last_listed.unwrap(),
parse_rfc3339("2020-01-27T18:52:09Z").unwrap()
);
mds[0].md().digest(),
&hex!("38c71329a87098cb341c46c9c62bd646622b4445f7eb985a0e6adb23a22ccf4f")
mds[0].md().ntor_key().as_bytes(),
&hex!("5e895d65304a3a1894616660143f7af5757fe08bc18045c7855ee8debb9e6c47")
assert!(mds[0].md().ipv4_policy().allows_port(993));
assert!(mds[0].md().ipv6_policy().allows_port(993));
assert!(!mds[0].md().ipv4_policy().allows_port(25));
assert!(!mds[0].md().ipv6_policy().allows_port(25));
mds[0].md().ed25519_id().as_bytes(),
&hex!("2d85fdc88e6c1bcfb46897fca1dba6d1354f93261d68a79e0b5bc170dd923084")
fn parse_family_ids() -> Result<()> {
let mds: Vec<AnnotatedMicrodesc> =
MicrodescReader::new(TESTDATA4, &AllowAnnotations::AnnotationsNotAllowed)?
assert_eq!(mds.len(), 2);
let md0 = mds[0].md();
let md1 = mds[1].md();
assert_eq!(md0.family_ids().len(), 0);
md1.family_ids(),
&[
"ed25519:dXMgdGhlIHRyaXVtcGguICAgIC1UaG9tYXMgUGFpbmU"
.parse()
.unwrap(),
"other:Example".parse().unwrap()
]
assert!(matches!(md1.family_ids()[0], RelayFamilyId::Ed25519(_)));
fn test_bad() {
use crate::types::policy::PolicyError;
use crate::Pos;
fn check(fname: &str, e: &Error) {
let content = read_bad(fname);
let res = Microdesc::parse(&content);
assert!(res.is_err());
assert_eq!(&res.err().unwrap(), e);
check(
"wrong-start",
&EK::WrongStartingToken
.with_msg("family")
.at_pos(Pos::from_line(1, 1)),
"bogus-policy",
&EK::BadPolicy
.at_pos(Pos::from_line(9, 1))
.with_source(PolicyError::InvalidPort),
check("wrong-id", &EK::MissingToken.with_msg("id ed25519"));
fn test_recover() -> Result<()> {
let mut data = read_bad("wrong-start");
data += TESTDATA;
data += &read_bad("wrong-id");
let res: Vec<Result<_>> =
MicrodescReader::new(&data, &AllowAnnotations::AnnotationsAllowed)?.collect();
assert_eq!(res.len(), 3);
assert!(res[0].is_err());
assert!(res[1].is_ok());
assert!(res[2].is_err());