Tor 0.4.9.0-alpha-dev
microdesc.c
Go to the documentation of this file.
1/* Copyright (c) 2009-2021, The Tor Project, Inc. */
2/* See LICENSE for licensing information */
3
4/**
5 * \file microdesc.c
6 *
7 * \brief Implements microdescriptors -- an abbreviated description of
8 * less-frequently-changing router information.
9 */
10
11#include "core/or/or.h"
12
13#include "lib/fdio/fdio.h"
14
15#include "app/config/config.h"
17#include "core/or/policies.h"
31
36
37#ifdef HAVE_FCNTL_H
38#include <fcntl.h>
39#endif
40#ifdef HAVE_SYS_STAT_H
41#include <sys/stat.h>
42#endif
43
44/** A data structure to hold a bunch of cached microdescriptors. There are
45 * two active files in the cache: a "cache file" that we mmap, and a "journal
46 * file" that we append to. Periodically, we rebuild the cache file to hold
47 * only the microdescriptors that we want to keep */
49 /** Map from sha256-digest to microdesc_t for every microdesc_t in the
50 * cache. */
51 HT_HEAD(microdesc_map, microdesc_t) map;
52
53 /** Name of the cache file. */
54 char *cache_fname;
55 /** Name of the journal file. */
56 char *journal_fname;
57 /** Mmap'd contents of the cache file, or NULL if there is none. */
58 tor_mmap_t *cache_content;
59 /** Number of bytes used in the journal file. */
60 size_t journal_len;
61 /** Number of bytes in descriptors removed as too old. */
62 size_t bytes_dropped;
63
64 /** Total bytes of microdescriptor bodies we have added to this cache */
65 uint64_t total_len_seen;
66 /** Total number of microdescriptors we have added to this cache */
67 unsigned n_seen;
68
69 /** True iff we have loaded this cache from disk ever. */
70 int is_loaded;
71};
72
74static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
75 const char *activity);
76
77/** Helper: computes a hash of <b>md</b> to place it in a hash table. */
78static inline unsigned int
80{
81 return (unsigned) siphash24g(md->digest, sizeof(md->digest));
82}
83
84/** Helper: compares <b>a</b> and <b>b</b> for equality for hash-table
85 * purposes. */
86static inline int
88{
89 return tor_memeq(a->digest, b->digest, DIGEST256_LEN);
90}
91
92HT_PROTOTYPE(microdesc_map, microdesc_t, node,
94HT_GENERATE2(microdesc_map, microdesc_t, node,
97
98/************************* md fetch fail cache *****************************/
99
100/* If we end up with too many outdated dirservers, something probably went
101 * wrong so clean up the list. */
102#define TOO_MANY_OUTDATED_DIRSERVERS 30
103
104/** List of dirservers with outdated microdesc information. The smartlist is
105 * filled with the hex digests of outdated dirservers. */
107
108/** Note that we failed to fetch a microdescriptor from the relay with
109 * <b>relay_digest</b> (of size DIGEST_LEN). */
110void
111microdesc_note_outdated_dirserver(const char *relay_digest)
112{
113 char relay_hexdigest[HEX_DIGEST_LEN+1];
114
115 /* If we have a reasonably live consensus, then most of our dirservers should
116 * still be caching all the microdescriptors in it. Reasonably live
117 * consensuses are up to a day old (or a day in the future). But
118 * microdescriptors expire 7 days after the last consensus that referenced
119 * them. */
121 FLAV_MICRODESC)) {
122 return;
123 }
124
127 }
128
130
131 /* If the list grows too big, clean it up */
132 if (smartlist_len(outdated_dirserver_list) > TOO_MANY_OUTDATED_DIRSERVERS) {
133 log_info(LD_GENERAL,"Too many outdated directory servers (%d). Resetting.",
134 smartlist_len(outdated_dirserver_list));
136 }
137
138 /* Turn the binary relay digest to a hex since smartlists have better support
139 * for strings than digests. */
140 base16_encode(relay_hexdigest,sizeof(relay_hexdigest),
141 relay_digest, DIGEST_LEN);
142
143 /* Make sure we don't add a dirauth as an outdated dirserver */
144 if (router_get_trusteddirserver_by_digest(relay_digest)) {
145 log_info(LD_GENERAL, "Auth %s gave us outdated dirinfo.", relay_hexdigest);
146 return;
147 }
148
149 /* Don't double-add outdated dirservers */
151 return;
152 }
153
154 /* Add it to the list of outdated dirservers */
156
157 log_info(LD_GENERAL, "Noted %s as outdated md dirserver", relay_hexdigest);
158}
159
160/** Return True if the relay with <b>relay_digest</b> (size DIGEST_LEN) is an
161 * outdated dirserver */
162int
164{
165 char relay_hexdigest[HEX_DIGEST_LEN+1];
166
168 return 0;
169 }
170
171 /* Convert identity digest to hex digest */
172 base16_encode(relay_hexdigest, sizeof(relay_hexdigest),
173 relay_digest, DIGEST_LEN);
174
175 /* Last time we tried to fetch microdescs, was this directory mirror missing
176 * any mds we asked for? */
178 return 1;
179 }
180
181 return 0;
182}
183
184/** Reset the list of outdated dirservers. */
185void
187{
189 return;
190 }
191
194}
195
196/****************************************************************************/
197
198/** Write the body of <b>md</b> into <b>f</b>, with appropriate annotations.
199 * On success, return the total number of bytes written, and set
200 * *<b>annotation_len_out</b> to the number of bytes written as
201 * annotations. */
202static ssize_t
203dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
204{
205 ssize_t r = 0;
206 ssize_t written;
207 if (md->body == NULL) {
208 *annotation_len_out = 0;
209 return 0;
210 }
211 /* XXXX drops unknown annotations. */
212 if (md->last_listed) {
213 char buf[ISO_TIME_LEN+1];
214 char annotation[ISO_TIME_LEN+32];
216 tor_snprintf(annotation, sizeof(annotation), "@last-listed %s\n", buf);
217 if (write_all_to_fd(fd, annotation, strlen(annotation)) < 0) {
218 log_warn(LD_DIR,
219 "Couldn't write microdescriptor annotation: %s",
220 strerror(errno));
221 return -1;
222 }
223 r += strlen(annotation);
224 *annotation_len_out = r;
225 } else {
226 *annotation_len_out = 0;
227 }
228
229 md->off = tor_fd_getpos(fd);
230 warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
231 "dumping a microdescriptor");
232 written = write_all_to_fd(fd, md->body, md->bodylen);
233 if (written != (ssize_t)md->bodylen) {
234 written = written < 0 ? 0 : written;
235 log_warn(LD_DIR,
236 "Couldn't dump microdescriptor (wrote %ld out of %lu): %s",
237 (long)written, (unsigned long)md->bodylen,
238 strerror(errno));
239 return -1;
240 }
241 r += md->bodylen;
242 return r;
243}
244
245/** Holds a pointer to the current microdesc_cache_t object, or NULL if no
246 * such object has been allocated. */
248
249/** Return a pointer to the microdescriptor cache, loading it if necessary. */
252{
254 if (PREDICT_UNLIKELY(cache->is_loaded == 0)) {
256 }
257 return cache;
258}
259
260/** Return a pointer to the microdescriptor cache, creating (but not loading)
261 * it if necessary. */
262static microdesc_cache_t *
264{
265 if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) {
266 microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache));
267 HT_INIT(microdesc_map, &cache->map);
268 cache->cache_fname = get_cachedir_fname("cached-microdescs");
269 cache->journal_fname = get_cachedir_fname("cached-microdescs.new");
270 the_microdesc_cache = cache;
271 }
272 return the_microdesc_cache;
273}
274
275/* There are three sources of microdescriptors:
276 1) Generated by us while acting as a directory authority.
277 2) Loaded from the cache on disk.
278 3) Downloaded.
279*/
280
281/** Decode the microdescriptors from the string starting at <b>s</b> and
282 * ending at <b>eos</b>, and store them in <b>cache</b>. If <b>no_save</b>,
283 * mark them as non-writable to disk. If <b>where</b> is SAVED_IN_CACHE,
284 * leave their bodies as pointers to the mmap'd cache. If where is
285 * <b>SAVED_NOWHERE</b>, do not allow annotations. If listed_at is not -1,
286 * set the last_listed field of every microdesc to listed_at. If
287 * requested_digests is non-null, then it contains a list of digests we mean
288 * to allow, so we should reject any non-requested microdesc with a different
289 * digest, and alter the list to contain only the digests of those microdescs
290 * we didn't find.
291 * Return a newly allocated list of the added microdescriptors, or NULL */
294 const char *s, const char *eos, saved_location_t where,
295 int no_save, time_t listed_at,
296 smartlist_t *requested_digests256)
297{
298 void * const DIGEST_REQUESTED = (void*)1;
299 void * const DIGEST_RECEIVED = (void*)2;
300 void * const DIGEST_INVALID = (void*)3;
301
302 smartlist_t *descriptors, *added;
303 const int allow_annotations = (where != SAVED_NOWHERE);
304 smartlist_t *invalid_digests = smartlist_new();
305
306 descriptors = microdescs_parse_from_string(s, eos,
307 allow_annotations,
308 where, invalid_digests);
309 if (listed_at != (time_t)-1) {
310 SMARTLIST_FOREACH(descriptors, microdesc_t *, md,
311 md->last_listed = listed_at);
312 }
313 if (requested_digests256) {
314 digest256map_t *requested;
315 requested = digest256map_new();
316 /* Set requested[d] to DIGEST_REQUESTED for every md we requested. */
317 SMARTLIST_FOREACH(requested_digests256, const uint8_t *, cp,
318 digest256map_set(requested, cp, DIGEST_REQUESTED));
319 /* Set requested[d] to DIGEST_INVALID for every md we requested which we
320 * will never be able to parse. Remove the ones we didn't request from
321 * invalid_digests.
322 */
323 SMARTLIST_FOREACH_BEGIN(invalid_digests, uint8_t *, cp) {
324 if (digest256map_get(requested, cp)) {
325 digest256map_set(requested, cp, DIGEST_INVALID);
326 } else {
327 tor_free(cp);
328 SMARTLIST_DEL_CURRENT(invalid_digests, cp);
329 }
330 } SMARTLIST_FOREACH_END(cp);
331 /* Update requested[d] to 2 for the mds we asked for and got. Delete the
332 * ones we never requested from the 'descriptors' smartlist.
333 */
334 SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
335 if (digest256map_get(requested, (const uint8_t*)md->digest)) {
336 digest256map_set(requested, (const uint8_t*)md->digest,
337 DIGEST_RECEIVED);
338 } else {
339 log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Received non-requested microdesc");
340 microdesc_free(md);
341 SMARTLIST_DEL_CURRENT(descriptors, md);
342 }
343 } SMARTLIST_FOREACH_END(md);
344 /* Remove the ones we got or the invalid ones from requested_digests256.
345 */
346 SMARTLIST_FOREACH_BEGIN(requested_digests256, uint8_t *, cp) {
347 void *status = digest256map_get(requested, cp);
348 if (status == DIGEST_RECEIVED || status == DIGEST_INVALID) {
349 tor_free(cp);
350 SMARTLIST_DEL_CURRENT(requested_digests256, cp);
351 }
352 } SMARTLIST_FOREACH_END(cp);
353 digest256map_free(requested, NULL);
354 }
355
356 /* For every requested microdescriptor that was unparseable, mark it
357 * as not to be retried. */
358 if (smartlist_len(invalid_digests)) {
359 networkstatus_t *ns =
361 if (ns) {
362 SMARTLIST_FOREACH_BEGIN(invalid_digests, char *, d) {
363 routerstatus_t *rs =
365 if (rs && tor_memeq(d, rs->descriptor_digest, DIGEST256_LEN)) {
366 download_status_mark_impossible(&rs->dl_status);
367 }
368 } SMARTLIST_FOREACH_END(d);
369 }
370 }
371 SMARTLIST_FOREACH(invalid_digests, uint8_t *, d, tor_free(d));
372 smartlist_free(invalid_digests);
373
374 added = microdescs_add_list_to_cache(cache, descriptors, where, no_save);
375 smartlist_free(descriptors);
376 return added;
377}
378
379/** As microdescs_add_to_cache, but takes a list of microdescriptors instead of
380 * a string to decode. Frees any members of <b>descriptors</b> that it does
381 * not add. */
384 smartlist_t *descriptors, saved_location_t where,
385 int no_save)
386{
387 smartlist_t *added;
388 open_file_t *open_file = NULL;
389 int fd = -1;
390 // int n_added = 0;
391 ssize_t size = 0;
392
393 if (where == SAVED_NOWHERE && !no_save) {
394 fd = start_writing_to_file(cache->journal_fname,
395 OPEN_FLAGS_APPEND|O_BINARY,
396 0600, &open_file);
397 if (fd < 0) {
398 log_warn(LD_DIR, "Couldn't append to journal in %s: %s",
399 cache->journal_fname, strerror(errno));
400 }
401 }
402
403 added = smartlist_new();
404 SMARTLIST_FOREACH_BEGIN(descriptors, microdesc_t *, md) {
405 microdesc_t *md2;
406 md2 = HT_FIND(microdesc_map, &cache->map, md);
407 if (md2) {
408 /* We already had this one. */
409 if (md2->last_listed < md->last_listed)
410 md2->last_listed = md->last_listed;
411 microdesc_free(md);
412 if (where != SAVED_NOWHERE)
413 cache->bytes_dropped += size;
414 continue;
415 }
416
417 /* Okay, it's a new one. */
418 if (fd >= 0) {
419 size_t annotation_len;
420 size = dump_microdescriptor(fd, md, &annotation_len);
421 if (size < 0) {
422 /* we already warned in dump_microdescriptor */
423 abort_writing_to_file(open_file);
424 fd = -1;
425 } else {
426 md->saved_location = SAVED_IN_JOURNAL;
427 cache->journal_len += size;
428 }
429 } else {
430 md->saved_location = where;
431 }
432
433 md->no_save = no_save;
434
435 HT_INSERT(microdesc_map, &cache->map, md);
436 md->held_in_map = 1;
437 smartlist_add(added, md);
438 ++cache->n_seen;
439 cache->total_len_seen += md->bodylen;
440 } SMARTLIST_FOREACH_END(md);
441
442 if (fd >= 0) {
443 if (finish_writing_to_file(open_file) < 0) {
444 log_warn(LD_DIR, "Error appending to microdescriptor file: %s",
445 strerror(errno));
446 smartlist_clear(added);
447 return added;
448 }
449 }
450
451 {
453 if (ns && ns->flavor == FLAV_MICRODESC)
455 }
456
457 if (smartlist_len(added))
459
460 return added;
461}
462
463/** Remove every microdescriptor in <b>cache</b>. */
464void
466{
467 microdesc_t **entry, **next;
468
469 for (entry = HT_START(microdesc_map, &cache->map); entry; entry = next) {
470 microdesc_t *md = *entry;
471 next = HT_NEXT_RMV(microdesc_map, &cache->map, entry);
472 md->held_in_map = 0;
473 microdesc_free(md);
474 }
475 HT_CLEAR(microdesc_map, &cache->map);
476 if (cache->cache_content) {
477 int res = tor_munmap_file(cache->cache_content);
478 if (res != 0) {
479 log_warn(LD_FS,
480 "tor_munmap_file() failed clearing microdesc cache; "
481 "we are probably about to leak memory.");
482 /* TODO something smarter? */
483 }
484 cache->cache_content = NULL;
485 }
486 cache->total_len_seen = 0;
487 cache->n_seen = 0;
488 cache->bytes_dropped = 0;
489}
490
491static void
492warn_if_nul_found(const char *inp, size_t len, int64_t offset,
493 const char *activity)
494{
495 const char *nul_found = memchr(inp, 0, len);
496 if (BUG(nul_found)) {
497 log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
498 "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
499 activity, offset, (nul_found - inp), len);
500 const char *start_excerpt_at, *eos = inp + len;
501 if ((nul_found - inp) >= 16)
502 start_excerpt_at = nul_found - 16;
503 else
504 start_excerpt_at = inp;
505 size_t excerpt_len = MIN(32, eos - start_excerpt_at);
506 char tmp[65];
507 base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
508 log_warn(LD_BUG, " surrounding string: %s", tmp);
509 }
510}
511
512/** Reload the contents of <b>cache</b> from disk. If it is empty, load it
513 * for the first time. Return 0 on success, -1 on failure. */
514int
516{
517 struct stat st;
518 char *journal_content;
519 smartlist_t *added;
520 tor_mmap_t *mm;
521 int total = 0;
522
524
525 cache->is_loaded = 1;
526
527 mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
528 if (mm) {
529 warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
530 added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
531 SAVED_IN_CACHE, 0, -1, NULL);
532 if (added) {
533 total += smartlist_len(added);
534 smartlist_free(added);
535 }
536 }
537
538 journal_content = read_file_to_str(cache->journal_fname,
540 if (journal_content) {
541 cache->journal_len = strlen(journal_content);
542 warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
543 "reading microdesc journal");
544 added = microdescs_add_to_cache(cache, journal_content,
545 journal_content+st.st_size,
546 SAVED_IN_JOURNAL, 0, -1, NULL);
547 if (added) {
548 total += smartlist_len(added);
549 smartlist_free(added);
550 }
551 tor_free(journal_content);
552 }
553 log_info(LD_DIR, "Reloaded microdescriptor cache. Found %d descriptors.",
554 total);
555
556 microdesc_cache_rebuild(cache, 0 /* don't force */);
557
558 return 0;
559}
560
561/** By default, we remove any microdescriptors that have gone at least this
562 * long without appearing in a current consensus. */
563#define TOLERATE_MICRODESC_AGE (7*24*60*60)
564
565/** Remove all microdescriptors from <b>cache</b> that haven't been listed for
566 * a long time. Does not rebuild the cache on disk. If <b>cutoff</b> is
567 * positive, specifically remove microdescriptors that have been unlisted
568 * since <b>cutoff</b>. If <b>force</b> is true, remove microdescriptors even
569 * if we have no current live microdescriptor consensus.
570 */
571void
572microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
573{
574 microdesc_t **mdp, *victim;
575 int dropped=0, kept=0;
576 size_t bytes_dropped = 0;
577 time_t now = time(NULL);
578
579 /* If we don't know a reasonably live consensus, don't believe last_listed
580 * values: we might be starting up after being down for a while. */
581 if (! force &&
582 ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
583 return;
584
585 if (cutoff <= 0)
586 cutoff = now - TOLERATE_MICRODESC_AGE;
587
588 for (mdp = HT_START(microdesc_map, &cache->map); mdp != NULL; ) {
589 const int is_old = (*mdp)->last_listed < cutoff;
590 const unsigned held_by_nodes = (*mdp)->held_by_nodes;
591 if (is_old && !held_by_nodes) {
592 ++dropped;
593 victim = *mdp;
594 mdp = HT_NEXT_RMV(microdesc_map, &cache->map, mdp);
595 victim->held_in_map = 0;
596 bytes_dropped += victim->bodylen;
597 microdesc_free(victim);
598 } else {
599 if (is_old) {
600 /* It's old, but it has held_by_nodes set. That's not okay. */
601 /* Let's try to diagnose and fix #7164 . */
604 long networkstatus_age = -1;
605 const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
606 if (ns) {
607 networkstatus_age = now - ns->valid_after;
608 }
609 log_warn(LD_BUG, "Microdescriptor seemed very old "
610 "(last listed %d hours ago vs %d hour cutoff), but is still "
611 "marked as being held by %d node(s). I found %d node(s) "
612 "holding it. Current networkstatus is %ld hours old. "
613 "Hashtable badness is %d.",
614 (int)((now - (*mdp)->last_listed) / 3600),
615 (int)((now - cutoff) / 3600),
616 held_by_nodes,
617 smartlist_len(nodes),
618 networkstatus_age / 3600,
619 ht_badness);
620
621 SMARTLIST_FOREACH_BEGIN(nodes, const node_t *, node) {
622 const char *rs_match = "No RS";
623 const char *rs_present = "";
624 if (node->rs) {
625 if (tor_memeq(node->rs->descriptor_digest,
626 (*mdp)->digest, DIGEST256_LEN)) {
627 rs_match = "Microdesc digest in RS matches";
628 } else {
629 rs_match = "Microdesc digest in RS does not match";
630 }
631 if (ns) {
632 /* This should be impossible, but let's see! */
633 rs_present = " RS not present in networkstatus.";
635 if (rs == node->rs) {
636 rs_present = " RS okay in networkstatus.";
637 }
638 });
639 }
640 }
641 log_warn(LD_BUG, " [%d]: ID=%s. md=%p, rs=%p, ri=%p. %s.%s",
642 node_sl_idx,
643 hex_str(node->identity, DIGEST_LEN),
644 node->md, node->rs, node->ri, rs_match, rs_present);
645 } SMARTLIST_FOREACH_END(node);
646 smartlist_free(nodes);
647 (*mdp)->last_listed = now;
648 }
649
650 ++kept;
651 mdp = HT_NEXT(microdesc_map, &cache->map, mdp);
652 }
653 }
654
655 if (dropped) {
656 log_info(LD_DIR, "Removed %d/%d microdescriptors as old.",
657 dropped,dropped+kept);
658 cache->bytes_dropped += bytes_dropped;
659 }
660}
661
662static int
663should_rebuild_md_cache(microdesc_cache_t *cache)
664{
665 const size_t old_len =
666 cache->cache_content ? cache->cache_content->size : 0;
667 const size_t journal_len = cache->journal_len;
668 const size_t dropped = cache->bytes_dropped;
669
670 if (journal_len < 16384)
671 return 0; /* Don't bother, not enough has happened yet. */
672 if (dropped > (journal_len + old_len) / 3)
673 return 1; /* We could save 1/3 or more of the currently used space. */
674 if (journal_len > old_len / 2)
675 return 1; /* We should append to the regular file */
676
677 return 0;
678}
679
680/**
681 * Mark <b>md</b> as having no body, and release any storage previously held
682 * by its body.
683 */
684static void
686{
687 if (!md)
688 return;
689
691 tor_free(md->body);
692
693 md->off = 0;
695 md->body = NULL;
696 md->bodylen = 0;
697 md->no_save = 1;
698}
699
700/** Regenerate the main cache file for <b>cache</b>, clear the journal file,
701 * and update every microdesc_t in the cache with pointers to its new
702 * location. If <b>force</b> is true, do this unconditionally. If
703 * <b>force</b> is false, do it only if we expect to save space on disk. */
704int
706{
707 open_file_t *open_file;
708 int fd = -1, res;
709 microdesc_t **mdp;
710 smartlist_t *wrote;
711 ssize_t size;
712 off_t off = 0, off_real;
713 int orig_size, new_size;
714
715 if (cache == NULL) {
716 cache = the_microdesc_cache;
717 if (cache == NULL)
718 return 0;
719 }
720
721 /* Remove dead descriptors */
722 microdesc_cache_clean(cache, 0/*cutoff*/, 0/*force*/);
723
724 if (!force && !should_rebuild_md_cache(cache))
725 return 0;
726
727 log_info(LD_DIR, "Rebuilding the microdescriptor cache...");
728
729 orig_size = (int)(cache->cache_content ? cache->cache_content->size : 0);
730 orig_size += (int)cache->journal_len;
731
732 fd = start_writing_to_file(cache->cache_fname,
733 OPEN_FLAGS_REPLACE|O_BINARY,
734 0600, &open_file);
735 if (fd < 0)
736 return -1;
737
738 wrote = smartlist_new();
739
740 HT_FOREACH(mdp, microdesc_map, &cache->map) {
741 microdesc_t *md = *mdp;
742 size_t annotation_len;
743 if (md->no_save || !md->body)
744 continue;
745
746 size = dump_microdescriptor(fd, md, &annotation_len);
747 if (size < 0) {
749
750 /* rewind, in case it was a partial write. */
751 tor_fd_setpos(fd, off);
752 continue;
753 }
754 tor_assert(((size_t)size) == annotation_len + md->bodylen);
755 md->off = off + annotation_len;
756 off += size;
757 off_real = tor_fd_getpos(fd);
758 if (off_real != off) {
759 log_warn(LD_BUG, "Discontinuity in position in microdescriptor cache."
760 "By my count, I'm at %"PRId64
761 ", but I should be at %"PRId64,
762 (int64_t)(off), (int64_t)(off_real));
763 if (off_real >= 0)
764 off = off_real;
765 }
766 if (md->saved_location != SAVED_IN_CACHE) {
767 tor_free(md->body);
769 }
770 smartlist_add(wrote, md);
771 }
772
773 /* We must do this unmap _before_ we call finish_writing_to_file(), or
774 * windows will not actually replace the file. */
775 if (cache->cache_content) {
776 res = tor_munmap_file(cache->cache_content);
777 if (res != 0) {
778 log_warn(LD_FS,
779 "Failed to unmap old microdescriptor cache while rebuilding");
780 }
781 cache->cache_content = NULL;
782 }
783
784 if (finish_writing_to_file(open_file) < 0) {
785 log_warn(LD_DIR, "Error rebuilding microdescriptor cache: %s",
786 strerror(errno));
787 /* Okay. Let's prevent from making things worse elsewhere. */
788 cache->cache_content = NULL;
789 HT_FOREACH(mdp, microdesc_map, &cache->map) {
790 microdesc_t *md = *mdp;
791 if (md->saved_location == SAVED_IN_CACHE) {
793 }
794 }
795 smartlist_free(wrote);
796 return -1;
797 }
798
799 cache->cache_content = tor_mmap_file(cache->cache_fname);
800
801 if (!cache->cache_content && smartlist_len(wrote)) {
802 log_err(LD_DIR, "Couldn't map file that we just wrote to %s!",
803 cache->cache_fname);
804 smartlist_free(wrote);
805 return -1;
806 }
808 tor_assert(md->saved_location == SAVED_IN_CACHE);
809 md->body = (char*)cache->cache_content->data + md->off;
810 if (PREDICT_UNLIKELY(
811 md->bodylen < 9 || fast_memneq(md->body, "onion-key", 9) != 0)) {
812 /* XXXX once bug 2022 is solved, we can kill this block and turn it
813 * into just the tor_assert(fast_memeq) */
814 off_t avail = cache->cache_content->size - md->off;
815 char *bad_str;
816 tor_assert(avail >= 0);
817 bad_str = tor_strndup(md->body, MIN(128, (size_t)avail));
818 log_err(LD_BUG, "After rebuilding microdesc cache, offsets seem wrong. "
819 " At offset %d, I expected to find a microdescriptor starting "
820 " with \"onion-key\". Instead I got %s.",
821 (int)md->off, escaped(bad_str));
822 tor_free(bad_str);
823 tor_assert(fast_memeq(md->body, "onion-key", 9));
824 }
825 } SMARTLIST_FOREACH_END(md);
826
827 smartlist_free(wrote);
828
829 write_str_to_file(cache->journal_fname, "", 1);
830 cache->journal_len = 0;
831 cache->bytes_dropped = 0;
832
833 new_size = cache->cache_content ? (int)cache->cache_content->size : 0;
834 log_info(LD_DIR, "Done rebuilding microdesc cache. "
835 "Saved %d bytes; %d still used.",
836 orig_size-new_size, new_size);
837
838 return 0;
839}
840
841/** Make sure that the reference count of every microdescriptor in cache is
842 * accurate. */
843void
845{
846 microdesc_t **mdp;
848 return;
849
850 HT_FOREACH(mdp, microdesc_map, &the_microdesc_cache->map) {
851 microdesc_t *md = *mdp;
852 unsigned int found=0;
853 const smartlist_t *nodes = nodelist_get_list();
854 SMARTLIST_FOREACH(nodes, node_t *, node, {
855 if (node->md == md) {
856 ++found;
857 }
858 });
859 tor_assert(found == md->held_by_nodes);
860 }
861}
862
863/** Deallocate a single microdescriptor. Note: the microdescriptor MUST have
864 * previously been removed from the cache if it had ever been inserted. */
865void
866microdesc_free_(microdesc_t *md, const char *fname, int lineno)
867{
868 if (!md)
869 return;
870
871 /* Make sure that the microdesc was really removed from the appropriate data
872 structures. */
873 if (md->held_in_map) {
875 microdesc_t *md2 = HT_FIND(microdesc_map, &cache->map, md);
876 if (md2 == md) {
877 log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
878 "in microdesc_map", fname, lineno);
879 HT_REMOVE(microdesc_map, &cache->map, md);
880 } else {
881 log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_in_map "
882 "set, but microdesc was not in the map.", fname, lineno);
883 }
885 }
886 if (md->held_by_nodes) {
888 int found=0;
889 const smartlist_t *nodes = nodelist_get_list();
890 const int ht_badness = HT_REP_IS_BAD_(microdesc_map, &cache->map);
891 SMARTLIST_FOREACH(nodes, node_t *, node, {
892 if (node->md == md) {
893 ++found;
894 node->md = NULL;
895 }
896 });
897 if (found) {
898 log_warn(LD_BUG, "microdesc_free() called from %s:%d, but md was still "
899 "referenced %d node(s); held_by_nodes == %u, ht_badness == %d",
900 fname, lineno, found, md->held_by_nodes, ht_badness);
901 } else {
902 log_warn(LD_BUG, "microdesc_free() called from %s:%d with held_by_nodes "
903 "set to %u, but md was not referenced by any nodes. "
904 "ht_badness == %d",
905 fname, lineno, md->held_by_nodes, ht_badness);
906 }
908 }
909 //tor_assert(md->held_in_map == 0);
910 //tor_assert(md->held_by_nodes == 0);
911
914 if (md->body && md->saved_location != SAVED_IN_CACHE)
915 tor_free(md->body);
916
917 nodefamily_free(md->family);
918 short_policy_free(md->exit_policy);
919 short_policy_free(md->ipv6_exit_policy);
920
921 tor_free(md);
922}
923
924/** Free all storage held in the microdesc.c module. */
925void
927{
930 tor_free(the_microdesc_cache->cache_fname);
931 tor_free(the_microdesc_cache->journal_fname);
933 }
934
937 smartlist_free(outdated_dirserver_list);
938 }
939}
940
941/** If there is a microdescriptor in <b>cache</b> whose sha256 digest is
942 * <b>d</b>, return it. Otherwise return NULL. */
945{
946 microdesc_t *md, search;
947 if (!cache)
948 cache = get_microdesc_cache();
949 memcpy(search.digest, d, DIGEST256_LEN);
950 md = HT_FIND(microdesc_map, &cache->map, &search);
951 return md;
952}
953
954/** Return a smartlist of all the sha256 digest of the microdescriptors that
955 * are listed in <b>ns</b> but not present in <b>cache</b>. Returns pointers
956 * to internals of <b>ns</b>; you should not free the members of the resulting
957 * smartlist. Omit all microdescriptors whose digest appear in <b>skip</b>. */
960 int downloadable_only, digest256map_t *skip)
961{
962 smartlist_t *result = smartlist_new();
963 time_t now = time(NULL);
964 tor_assert(ns->flavor == FLAV_MICRODESC);
966 if (microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest))
967 continue;
968 if (downloadable_only &&
969 !download_status_is_ready(&rs->dl_status, now))
970 continue;
971 if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
972 continue;
973 if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
974 continue;
975 /* XXXX Also skip if we're a noncache and wouldn't use this router.
976 * XXXX NM Microdesc
977 */
978 smartlist_add(result, rs->descriptor_digest);
979 } SMARTLIST_FOREACH_END(rs);
980 return result;
981}
982
983/** Launch download requests for microdescriptors as appropriate.
984 *
985 * Specifically, we should launch download requests if we are configured to
986 * download mirodescriptors, and there are some microdescriptors listed in the
987 * current microdesc consensus that we don't have, and either we never asked
988 * for them, or we failed to download them but we're willing to retry.
989 */
990void
992{
993 const or_options_t *options = get_options();
994 networkstatus_t *consensus;
995 smartlist_t *missing;
996 digest256map_t *pending;
997
998 if (should_delay_dir_fetches(options, NULL))
999 return;
1001 return;
1002
1003 /* Give up if we don't have a reasonably live consensus. */
1004 consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
1005 if (!consensus)
1006 return;
1007
1008 if (!we_fetch_microdescriptors(options))
1009 return;
1010
1011 pending = digest256map_new();
1013
1014 missing = microdesc_list_missing_digest256(consensus,
1016 1,
1017 pending);
1018 digest256map_free(pending, NULL);
1019
1021 missing, NULL, now);
1022
1023 smartlist_free(missing);
1024}
1025
1026/** For every microdescriptor listed in the current microdescriptor consensus,
1027 * update its last_listed field to be at least as recent as the publication
1028 * time of the current microdescriptor consensus.
1029 */
1030void
1032{
1034 microdesc_t *md;
1035 networkstatus_t *ns =
1037
1038 if (! ns)
1039 return;
1040
1041 tor_assert(ns->flavor == FLAV_MICRODESC);
1042
1044 md = microdesc_cache_lookup_by_digest256(cache, rs->descriptor_digest);
1045 if (md && ns->valid_after > md->last_listed)
1046 md->last_listed = ns->valid_after;
1047 } SMARTLIST_FOREACH_END(rs);
1048}
1049
1050/** Return true iff we should prefer to use microdescriptors rather than
1051 * routerdescs for building circuits. */
1052int
1054{
1055 if (options->UseMicrodescriptors == 0)
1056 return 0; /* the user explicitly picked no */
1057 return 1; /* yes and auto both mean yes */
1058}
1059
1060/** Return true iff we should try to download microdescriptors at all. */
1061int
1063{
1064 if (directory_caches_dir_info(options))
1065 return 1;
1066 if (options->FetchUselessDescriptors)
1067 return 1;
1069}
1070
1071/** Return true iff we should try to download router descriptors at all. */
1072int
1074{
1075 if (directory_caches_dir_info(options))
1076 return 1;
1077 if (options->FetchUselessDescriptors)
1078 return 1;
1079 return ! we_use_microdescriptors_for_circuits(options);
1080}
1081
1082/** Return the consensus flavor we actually want to use to build circuits. */
1083MOCK_IMPL(int,
1085{
1087 return FLAV_MICRODESC;
1088 } else {
1089 return FLAV_NS;
1090 }
1091}
time_t approx_time(void)
Definition: approx_time.c:32
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
Header file for circuitbuild.c.
const or_options_t * get_options(void)
Definition: config.c:944
Header file for config.c.
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
int tor_memeq(const void *a, const void *b, size_t sz)
Definition: di_ops.c:107
#define fast_memeq(a, b, c)
Definition: di_ops.h:35
#define fast_memneq(a, b, c)
Definition: di_ops.h:42
#define DIGEST_LEN
Definition: digest_sizes.h:20
#define DIGEST256_LEN
Definition: digest_sizes.h:23
int dirclient_too_idle_to_fetch_descriptors(const or_options_t *options, time_t now)
Header for feature/dirclient/dirclient_modes.c.
Header file for directory.c.
#define DIR_PURPOSE_FETCH_MICRODESC
Definition: directory.h:65
dir_server_t * router_get_trusteddirserver_by_digest(const char *digest)
Definition: dirlist.c:160
Header file for dirlist.c.
int directory_caches_dir_info(const or_options_t *options)
Definition: dirserv.c:94
Header file for dirserv.c.
int download_status_is_ready(download_status_t *dls, time_t now)
Definition: dlstatus.c:380
void download_status_mark_impossible(download_status_t *dl)
Definition: dlstatus.c:392
Header file for dlstatus.c.
Header file for circuitbuild.c.
const char * escaped(const char *s)
Definition: escape.c:126
off_t tor_fd_getpos(int fd)
Definition: fdio.c:48
int tor_fd_setpos(int fd, off_t pos)
Definition: fdio.c:80
Header for fdio.c.
int write_str_to_file(const char *fname, const char *str, int bin)
Definition: files.c:275
#define RFTS_IGNORE_MISSING
Definition: files.h:101
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:466
int start_writing_to_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:318
ssize_t write_all_to_fd(int fd, const char *buf, size_t count)
Definition: files.c:163
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:474
HT_PROTOTYPE(hs_circuitmap_ht, circuit_t, hs_circuitmap_node, hs_circuit_hash_token, hs_circuits_have_same_token)
typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht
#define log_fn(severity, domain, args,...)
Definition: log.h:283
#define LD_FS
Definition: log.h:70
#define LD_BUG
Definition: log.h:86
#define LD_GENERAL
Definition: log.h:62
#define LD_DIR
Definition: log.h:88
void * tor_reallocarray_(void *ptr, size_t sz1, size_t sz2)
Definition: malloc.c:146
void tor_free_(void *mem)
Definition: malloc.c:227
#define tor_free(p)
Definition: malloc.h:56
static int microdesc_eq_(microdesc_t *a, microdesc_t *b)
Definition: microdesc.c:87
void microdesc_free_(microdesc_t *md, const char *fname, int lineno)
Definition: microdesc.c:866
void microdesc_reset_outdated_dirservers_list(void)
Definition: microdesc.c:186
void microdesc_free_all(void)
Definition: microdesc.c:926
int usable_consensus_flavor(void)
Definition: microdesc.c:1084
static unsigned int microdesc_hash_(microdesc_t *md)
Definition: microdesc.c:79
smartlist_t * microdescs_add_list_to_cache(microdesc_cache_t *cache, smartlist_t *descriptors, saved_location_t where, int no_save)
Definition: microdesc.c:383
microdesc_t * microdesc_cache_lookup_by_digest256(microdesc_cache_t *cache, const char *d)
Definition: microdesc.c:944
int we_fetch_router_descriptors(const or_options_t *options)
Definition: microdesc.c:1073
void microdesc_note_outdated_dirserver(const char *relay_digest)
Definition: microdesc.c:111
smartlist_t * microdescs_add_to_cache(microdesc_cache_t *cache, const char *s, const char *eos, saved_location_t where, int no_save, time_t listed_at, smartlist_t *requested_digests256)
Definition: microdesc.c:293
void microdesc_check_counts(void)
Definition: microdesc.c:844
microdesc_cache_t * get_microdesc_cache(void)
Definition: microdesc.c:251
static microdesc_cache_t * get_microdesc_cache_noload(void)
Definition: microdesc.c:263
int microdesc_cache_rebuild(microdesc_cache_t *cache, int force)
Definition: microdesc.c:705
static smartlist_t * outdated_dirserver_list
Definition: microdesc.c:106
void microdesc_cache_clear(microdesc_cache_t *cache)
Definition: microdesc.c:465
static ssize_t dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
Definition: microdesc.c:203
int microdesc_cache_reload(microdesc_cache_t *cache)
Definition: microdesc.c:515
static void microdesc_wipe_body(microdesc_t *md)
Definition: microdesc.c:685
#define TOLERATE_MICRODESC_AGE
Definition: microdesc.c:563
void update_microdesc_downloads(time_t now)
Definition: microdesc.c:991
smartlist_t * microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache, int downloadable_only, digest256map_t *skip)
Definition: microdesc.c:959
int microdesc_relay_is_outdated_dirserver(const char *relay_digest)
Definition: microdesc.c:163
void update_microdescs_from_networkstatus(time_t now)
Definition: microdesc.c:1031
static microdesc_cache_t * the_microdesc_cache
Definition: microdesc.c:247
int we_fetch_microdescriptors(const or_options_t *options)
Definition: microdesc.c:1062
int we_use_microdescriptors_for_circuits(const or_options_t *options)
Definition: microdesc.c:1053
void microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
Definition: microdesc.c:572
Header file for microdesc.c.
smartlist_t * microdescs_parse_from_string(const char *s, const char *eos, int allow_annotations, saved_location_t where, smartlist_t *invalid_digests_out)
Header file for microdesc_parse.c.
Microdescriptor structure.
networkstatus_t * networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
networkstatus_t * networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
networkstatus_t * networkstatus_get_latest_consensus(void)
routerstatus_t * router_get_mutable_consensus_status_by_descriptor_digest(networkstatus_t *consensus, const char *digest)
int should_delay_dir_fetches(const or_options_t *options, const char **msg_out)
Header file for networkstatus.c.
Networkstatus consensus/vote structure.
Node information structure.
Header file for nodefamily.c.
void router_dir_info_changed(void)
Definition: nodelist.c:2448
const smartlist_t * nodelist_get_list(void)
Definition: nodelist.c:1047
node_t * nodelist_add_microdesc(microdesc_t *md)
Definition: nodelist.c:635
smartlist_t * nodelist_find_nodes_with_microdesc(const microdesc_t *md)
Definition: nodelist.c:863
Header file for nodelist.c.
Master header file for Tor-specific functionality.
saved_location_t
Definition: or.h:623
@ SAVED_IN_JOURNAL
Definition: or.h:637
@ SAVED_NOWHERE
Definition: or.h:626
@ SAVED_IN_CACHE
Definition: or.h:630
Header file for policies.c.
int tor_snprintf(char *str, size_t size, const char *format,...)
Definition: printf.c:27
Header file for router.c.
void list_pending_microdesc_downloads(digest256map_t *result)
Definition: routerlist.c:2428
void launch_descriptor_downloads(int purpose, smartlist_t *downloadable, const routerstatus_t *source, time_t now)
Definition: routerlist.c:2552
Header file for routerlist.c.
Routerstatus (consensus entry) structure.
int smartlist_contains_string(const smartlist_t *sl, const char *element)
Definition: smartlist.c:93
void smartlist_add_strdup(struct smartlist_t *sl, const char *string)
smartlist_t * smartlist_new(void)
void smartlist_add(smartlist_t *sl, void *element)
void smartlist_clear(smartlist_t *sl)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
#define SMARTLIST_DEL_CURRENT(sl, var)
unsigned int held_by_nodes
Definition: microdesc_st.h:46
char * body
Definition: microdesc_st.h:58
struct short_policy_t * exit_policy
Definition: microdesc_st.h:77
unsigned int held_in_map
Definition: microdesc_st.h:42
saved_location_bitfield_t saved_location
Definition: microdesc_st.h:38
unsigned int no_save
Definition: microdesc_st.h:40
time_t last_listed
Definition: microdesc_st.h:36
struct curve25519_public_key_t * onion_curve25519_pkey
Definition: microdesc_st.h:67
char digest[DIGEST256_LEN]
Definition: microdesc_st.h:62
size_t bodylen
Definition: microdesc_st.h:60
struct nodefamily_t * family
Definition: microdesc_st.h:75
struct ed25519_public_key_t * ed25519_identity_pkey
Definition: microdesc_st.h:69
struct short_policy_t * ipv6_exit_policy
Definition: microdesc_st.h:79
smartlist_t * routerstatus_list
consensus_flavor_t flavor
Definition: node_st.h:34
int FetchUselessDescriptors
int UseMicrodescriptors
char descriptor_digest[DIGEST256_LEN]
size_t size
Definition: mmap.h:27
const char * data
Definition: mmap.h:26
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:326
#define tor_assert(expr)
Definition: util_bug.h:103
#define tor_fragile_assert()
Definition: util_bug.h:278
int fast_mem_is_zero(const char *mem, size_t len)
Definition: util_string.c:76