Tor  0.4.8.0-alpha-dev
rephist.c
Go to the documentation of this file.
1 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
2  * Copyright (c) 2007-2021, The Tor Project, Inc. */
3 /* See LICENSE for licensing information */
4 
5 /**
6  * \file rephist.c
7  * \brief Basic history and performance-tracking functionality.
8  *
9  * Basic history and performance-tracking functionality to remember
10  * which servers have worked in the past, how much bandwidth we've
11  * been using, which ports we tend to want, and so on; further,
12  * exit port statistics, cell statistics, and connection statistics.
13  *
14  * The history and information tracked in this module could sensibly be
15  * divided into several categories:
16  *
17  * <ul><li>Statistics used by authorities to remember the uptime and
18  * stability information about various relays, including "uptime",
19  * "weighted fractional uptime" and "mean time between failures".
20  *
21  * <li>Predicted ports, used by clients to remember how long it's been
22  * since they opened an exit connection to each given target
23  * port. Clients use this information in order to try to keep circuits
24  * open to exit nodes that can connect to the ports that they care
25  * about. (The predicted ports mechanism also handles predicted circuit
26  * usage that _isn't_ port-specific, such as resolves, internal circuits,
27  * and so on.)
28  *
29  * <li>Public key operation counters, for tracking how many times we've
30  * done each public key operation. (This is unmaintained and we should
31  * remove it.)
32  *
33  * <li>Exit statistics by port, used by exits to keep track of the
34  * number of streams and bytes they've served at each exit port, so they
35  * can generate their exit-kibibytes-{read,written} and
36  * exit-streams-opened statistics.
37  *
38  * <li>Circuit stats, used by relays instances to tract circuit
39  * queue fullness and delay over time, and generate cell-processed-cells,
40  * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile
41  * statistics.
42  *
43  * <li>Descriptor serving statistics, used by directory caches to track
44  * how many descriptors they've served.
45  *
46  * <li>Onion handshake statistics, used by relays to count how many
47  * TAP and ntor handshakes they've handled.
48  *
49  * <li>Hidden service statistics, used by relays to count rendezvous
50  * traffic and HSDir-stored descriptors.
51  *
52  * <li>Link protocol statistics, used by relays to count how many times
53  * each link protocol has been used.
54  *
55  * </ul>
56  *
57  * The entry points for this module are scattered throughout the
58  * codebase. Sending data, receiving data, connecting to a relay,
59  * losing a connection to a relay, and so on can all trigger a change in
60  * our current stats. Relays also invoke this module in order to
61  * extract their statistics when building routerinfo and extrainfo
62  * objects in router.c.
63  *
64  * TODO: This module should be broken up.
65  *
66  * (The "rephist" name originally stood for "reputation and history". )
67  **/
68 
69 #define REPHIST_PRIVATE
70 #include "core/or/or.h"
71 #include "app/config/config.h"
73 #include "core/or/circuitlist.h"
74 #include "core/or/connection_or.h"
80 #include "feature/stats/rephist.h"
81 #include "lib/container/order.h"
83 #include "lib/math/laplace.h"
84 
86 #include "core/or/or_circuit_st.h"
87 
88 #include <event2/dns.h>
89 
90 #ifdef HAVE_FCNTL_H
91 #include <fcntl.h>
92 #endif
93 
94 /** Total number of bytes currently allocated in fields used by rephist.c. */
96 /** Number of or_history_t objects currently allocated. */
97 uint32_t rephist_total_num=0;
98 
99 /** If the total weighted run count of all runs for a router ever falls
100  * below this amount, the router can be treated as having 0 MTBF. */
101 #define STABILITY_EPSILON 0.0001
102 /** Value by which to discount all old intervals for MTBF purposes. This
103  * is compounded every STABILITY_INTERVAL. */
104 #define STABILITY_ALPHA 0.95
105 /** Interval at which to discount all old intervals for MTBF purposes. */
106 #define STABILITY_INTERVAL (12*60*60)
107 /* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an
108  * interval that just ended counts twice as much as one that ended a week ago,
109  * 20X as much as one that ended a month ago, and routers that have had no
110  * uptime data for about half a year will get forgotten.) */
111 
112 /** History of an OR. */
113 typedef struct or_history_t {
114  /** When did we start tracking this OR? */
115  time_t since;
116  /** When did we most recently note a change to this OR? */
117  time_t changed;
118 
119  /** The address at which we most recently connected to this OR
120  * successfully. */
122 
123  /** The port at which we most recently connected to this OR successfully */
125 
126  /* === For MTBF tracking: */
127  /** Weighted sum total of all times that this router has been online.
128  */
129  unsigned long weighted_run_length;
130  /** If the router is now online (according to stability-checking rules),
131  * when did it come online? */
132  time_t start_of_run;
133  /** Sum of weights for runs in weighted_run_length. */
135  /* === For fractional uptime tracking: */
136  time_t start_of_downtime;
137  unsigned long weighted_uptime;
138  unsigned long total_weighted_time;
139 } or_history_t;
140 
141 /**
142  * This structure holds accounting needed to calculate the padding overhead.
143  */
144 typedef struct padding_counts_t {
145  /** Total number of cells we have received, including padding */
146  uint64_t read_cell_count;
147  /** Total number of cells we have sent, including padding */
149  /** Total number of CELL_PADDING cells we have received */
151  /** Total number of CELL_PADDING cells we have sent */
153  /** Total number of read cells on padding-enabled conns */
155  /** Total number of sent cells on padding-enabled conns */
157  /** Total number of read CELL_PADDING cells on padding-enabled cons */
159  /** Total number of sent CELL_PADDING cells on padding-enabled cons */
161  /** Total number of RELAY_DROP cells we have received */
163  /** Total number of RELAY_DROP cells we have sent */
165  /** The maximum number of padding timers we've seen in 24 hours */
167  /** When did we first copy padding_current into padding_published? */
168  char first_published_at[ISO_TIME_LEN+1];
170 
171 /** Holds the current values of our padding statistics.
172  * It is not published until it is transferred to padding_published. */
174 
175 /** Remains fixed for a 24 hour period, and then is replaced
176  * by a redacted copy of padding_current */
178 
179 /** When did we last multiply all routers' weighted_run_length and
180  * total_run_weights by STABILITY_ALPHA? */
181 static time_t stability_last_downrated = 0;
182 
183 /** */
184 static time_t started_tracking_stability = 0;
185 
186 /** Map from hex OR identity digest to or_history_t. */
187 static digestmap_t *history_map = NULL;
188 
189 /** Represents a state of overload stats.
190  *
191  * All the timestamps in this structure have already been rounded down to the
192  * nearest hour. */
193 typedef struct {
194  /* When did we last experience a general overload? */
195  time_t overload_general_time;
196 
197  /* When did we last experience a bandwidth-related overload? */
198  time_t overload_ratelimits_time;
199  /* How many times have we gone off the our read limits? */
200  uint64_t overload_read_count;
201  /* How many times have we gone off the our write limits? */
202  uint64_t overload_write_count;
203 
204  /* When did we last experience a file descriptor exhaustion? */
205  time_t overload_fd_exhausted_time;
206  /* How many times have we experienced a file descriptor exhaustion? */
207  uint64_t overload_fd_exhausted;
209 
210 /** Current state of overload stats */
212 
213 /** Counters to count the number of times we've reached an overload for the
214  * global connection read/write limit. Reported on the MetricsPort. */
215 static uint64_t stats_n_read_limit_reached = 0;
216 static uint64_t stats_n_write_limit_reached = 0;
217 
218 /** Total number of times we've reached TCP port exhaustion. */
219 static uint64_t stats_n_tcp_exhaustion = 0;
220 
221 /***** DNS statistics *****/
222 
223 /** Overload DNS statistics. The information in this object is used to assess
224  * if, due to DNS errors, we should emit a general overload signal or not.
225  *
226  * NOTE: This structure is _not_ per DNS query type like the statistics below
227  * because of a libevent bug
228  * (https://github.com/libevent/libevent/issues/1219), on error, the type is
229  * not propagated up back to the user and so we need to keep our own stats for
230  * the overload signal. */
231 typedef struct {
232  /** Total number of DNS request seen at an Exit. They might not all end
233  * successfully or might even be lost by tor. This counter is incremented
234  * right before the DNS request is initiated. */
235  uint64_t stats_n_request;
236 
237  /** When is the next assessment time of the general overload for DNS errors.
238  * Once this time is reached, all stats are reset and this time is set to the
239  * next assessment time. */
242 
243 /** Keep track of the DNS requests for the general overload state. */
245 
246 /** Represents the statistics of DNS queries seen if it is an Exit. */
247 typedef struct {
248  /* Total number of DNS errors found in RFC 1035 (from 0 to 5 code). */
249  uint64_t stats_n_error_none; /* 0 */
250  uint64_t stats_n_error_format; /* 1 */
251  uint64_t stats_n_error_serverfailed; /* 2 */
252  uint64_t stats_n_error_notexist; /* 3 */
253  uint64_t stats_n_error_notimpl; /* 4 */
254  uint64_t stats_n_error_refused; /* 5 */
255 
256  /* Total number of DNS errors specific to libevent. */
257  uint64_t stats_n_error_truncated; /* 65 */
258  uint64_t stats_n_error_unknown; /* 66 */
259  uint64_t stats_n_error_tor_timeout; /* 67 */
260  uint64_t stats_n_error_shutdown; /* 68 */
261  uint64_t stats_n_error_cancel; /* 69 */
262  uint64_t stats_n_error_nodata; /* 70 */
263 
264  /* Total number of DNS request seen at an Exit. They might not all end
265  * successfully or might even be lost by tor. This counter is incremented
266  * right before the DNS request is initiated. */
267  uint64_t stats_n_request;
268 } dns_stats_t;
269 
270 /* This is disabled because of the libevent bug where on error we don't get the
271  * DNS query type back. Once it is fixed, we can re-enable this. */
272 #if 0
273 /** DNS statistics store for each DNS record type for which tor supports only
274  * three at the moment: A, PTR and AAAA. */
275 static dns_stats_t dns_A_stats;
276 static dns_stats_t dns_PTR_stats;
277 static dns_stats_t dns_AAAA_stats;
278 #endif
279 
280 /** DNS query statistics store. It covers all type of queries. */
282 
283 /** Return the point to the DNS statistics store. Ignore the type for now
284  * because of a libevent problem. */
285 static inline dns_stats_t *
286 get_dns_stats_by_type(const int type)
287 {
288  (void) type;
289  return &dns_all_stats;
290 }
291 
292 #if 0
293 /** From a libevent record type, return a pointer to the corresponding DNS
294  * statistics store. NULL is returned if the type is unhandled. */
295 static inline dns_stats_t *
296 get_dns_stats_by_type(const int type)
297 {
298  switch (type) {
299  case DNS_IPv4_A:
300  return &dns_A_stats;
301  case DNS_PTR:
302  return &dns_PTR_stats;
303  case DNS_IPv6_AAAA:
304  return &dns_AAAA_stats;
305  default:
306  return NULL;
307  }
308 }
309 #endif
310 
311 /** Return the DNS error count for the given libevent DNS type and error code.
312  * The possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */
313 uint64_t
314 rep_hist_get_n_dns_error(int type, uint8_t error)
315 {
316  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
317  if (BUG(!dns_stats)) {
318  return 0;
319  }
320 
321  switch (error) {
322  case DNS_ERR_NONE:
323  return dns_stats->stats_n_error_none;
324  case DNS_ERR_FORMAT:
325  return dns_stats->stats_n_error_format;
326  case DNS_ERR_SERVERFAILED:
327  return dns_stats->stats_n_error_serverfailed;
328  case DNS_ERR_NOTEXIST:
329  return dns_stats->stats_n_error_notexist;
330  case DNS_ERR_NOTIMPL:
331  return dns_stats->stats_n_error_notimpl;
332  case DNS_ERR_REFUSED:
333  return dns_stats->stats_n_error_refused;
334  case DNS_ERR_TRUNCATED:
335  return dns_stats->stats_n_error_truncated;
336  case DNS_ERR_UNKNOWN:
337  return dns_stats->stats_n_error_unknown;
338  case DNS_ERR_TIMEOUT:
339  return dns_stats->stats_n_error_tor_timeout;
340  case DNS_ERR_SHUTDOWN:
341  return dns_stats->stats_n_error_shutdown;
342  case DNS_ERR_CANCEL:
343  return dns_stats->stats_n_error_cancel;
344  case DNS_ERR_NODATA:
345  return dns_stats->stats_n_error_nodata;
346  default:
347  /* Unhandled code sent back by libevent. */
348  return 0;
349  }
350 }
351 
352 /** Return the total number of DNS request seen for the given libevent DNS
353  * record type. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA. */
354 uint64_t
356 {
357  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
358  if (BUG(!dns_stats)) {
359  return 0;
360  }
361  return dns_stats->stats_n_request;
362 }
363 
364 /** Note a DNS error for the given given libevent DNS record type and error
365  * code. Possible types are: DNS_IPv4_A, DNS_PTR, DNS_IPv6_AAAA.
366  *
367  * NOTE: Libevent is _not_ returning the type in case of an error and so if
368  * error is anything but DNS_ERR_NONE, the type is not usable and set to 0.
369  *
370  * See: https://gitlab.torproject.org/tpo/core/tor/-/issues/40490 */
371 void
372 rep_hist_note_dns_error(int type, uint8_t error)
373 {
375 
376  /* Again, the libevent bug (see function comment), for an error that is
377  * anything but DNS_ERR_NONE, the type is always 0 which means that we don't
378  * have a DNS stat object for it so this code will do nothing until libevent
379  * is fixed. */
380  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
381  /* Unsupported DNS query type. */
382  if (!dns_stats) {
383  return;
384  }
385 
386  switch (error) {
387  case DNS_ERR_NONE:
388  dns_stats->stats_n_error_none++;
389  break;
390  case DNS_ERR_FORMAT:
391  dns_stats->stats_n_error_format++;
392  break;
393  case DNS_ERR_SERVERFAILED:
394  dns_stats->stats_n_error_serverfailed++;
395  break;
396  case DNS_ERR_NOTEXIST:
397  dns_stats->stats_n_error_notexist++;
398  break;
399  case DNS_ERR_NOTIMPL:
400  dns_stats->stats_n_error_notimpl++;
401  break;
402  case DNS_ERR_REFUSED:
403  dns_stats->stats_n_error_refused++;
404  break;
405  case DNS_ERR_TRUNCATED:
406  dns_stats->stats_n_error_truncated++;
407  break;
408  case DNS_ERR_UNKNOWN:
409  dns_stats->stats_n_error_unknown++;
410  break;
411  case DNS_ERR_TIMEOUT:
412  dns_stats->stats_n_error_tor_timeout++;
413  break;
414  case DNS_ERR_SHUTDOWN:
415  dns_stats->stats_n_error_shutdown++;
416  break;
417  case DNS_ERR_CANCEL:
418  dns_stats->stats_n_error_cancel++;
419  break;
420  case DNS_ERR_NODATA:
421  dns_stats->stats_n_error_nodata++;
422  break;
423  default:
424  /* Unhandled code sent back by libevent. */
425  break;
426  }
427 }
428 
429 /** Note a DNS request for the given given libevent DNS record type. */
430 void
432 {
433  dns_stats_t *dns_stats = get_dns_stats_by_type(type);
434  if (BUG(!dns_stats)) {
435  return;
436  }
437  dns_stats->stats_n_request++;
438 }
439 
440 /***** END of DNS statistics *****/
441 
442 /** Return true if this overload happened within the last `n_hours`. */
443 static bool
444 overload_happened_recently(time_t overload_time, int n_hours)
445 {
446  /* An overload is relevant if it happened in the last 72 hours */
447  if (overload_time > approx_time() - 3600 * n_hours) {
448  return true;
449  }
450  return false;
451 }
452 
453 /* The current version of the overload stats version */
454 #define OVERLOAD_STATS_VERSION 1
455 
456 /** Return the stats_n_read_limit_reached counter. */
457 uint64_t
459 {
461 }
462 
463 /** Return the stats_n_write_limit_reached counter. */
464 uint64_t
466 {
467  return stats_n_write_limit_reached;
468 }
469 
470 /** Returns an allocated string for server descriptor for publising information
471  * on whether we are overloaded or not. */
472 char *
474 {
475  char *result = NULL;
476  char tbuf[ISO_TIME_LEN+1];
477 
478  /* Encode the general overload */
479  if (overload_happened_recently(overload_stats.overload_general_time, 72)) {
480  format_iso_time(tbuf, overload_stats.overload_general_time);
481  tor_asprintf(&result, "overload-general %d %s\n",
482  OVERLOAD_STATS_VERSION, tbuf);
483  }
484 
485  return result;
486 }
487 
488 /** Returns an allocated string for extra-info documents for publishing
489  * overload statistics. */
490 char *
492 {
493  char *result = NULL;
494  smartlist_t *chunks = smartlist_new();
495  char tbuf[ISO_TIME_LEN+1];
496 
497  /* Add bandwidth-related overloads */
498  if (overload_happened_recently(overload_stats.overload_ratelimits_time,24)) {
499  const or_options_t *options = get_options();
500  format_iso_time(tbuf, overload_stats.overload_ratelimits_time);
501  smartlist_add_asprintf(chunks,
502  "overload-ratelimits %d %s %" PRIu64 " %" PRIu64
503  " %" PRIu64 " %" PRIu64 "\n",
504  OVERLOAD_STATS_VERSION, tbuf,
505  options->BandwidthRate, options->BandwidthBurst,
506  overload_stats.overload_read_count,
507  overload_stats.overload_write_count);
508  }
509 
510  /* Finally file descriptor overloads */
512  overload_stats.overload_fd_exhausted_time, 72)) {
513  format_iso_time(tbuf, overload_stats.overload_fd_exhausted_time);
514  smartlist_add_asprintf(chunks, "overload-fd-exhausted %d %s\n",
515  OVERLOAD_STATS_VERSION, tbuf);
516  }
517 
518  /* Bail early if we had nothing to write */
519  if (smartlist_len(chunks) == 0) {
520  goto done;
521  }
522 
523  result = smartlist_join_strings(chunks, "", 0, NULL);
524 
525  done:
526  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
527  smartlist_free(chunks);
528  return result;
529 }
530 
531 /** Round down the time in `a` to the beginning of the current hour */
532 #define SET_TO_START_OF_HOUR(a) STMT_BEGIN \
533  (a) = approx_time() - (approx_time() % 3600); \
534 STMT_END
535 
536 /** Note down an overload event of type `overload`. */
537 void
539 {
540  static time_t last_read_counted = 0;
541  static time_t last_write_counted = 0;
542 
543  switch (overload) {
544  case OVERLOAD_GENERAL:
545  SET_TO_START_OF_HOUR(overload_stats.overload_general_time);
546  break;
547  case OVERLOAD_READ: {
549  SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time);
550  if (approx_time() >= last_read_counted + 60) { /* Count once a minute */
551  overload_stats.overload_read_count++;
552  last_read_counted = approx_time();
553  }
554  break;
555  }
556  case OVERLOAD_WRITE: {
557  stats_n_write_limit_reached++;
558  SET_TO_START_OF_HOUR(overload_stats.overload_ratelimits_time);
559  if (approx_time() >= last_write_counted + 60) { /* Count once a minute */
560  overload_stats.overload_write_count++;
561  last_write_counted = approx_time();
562  }
563  break;
564  }
565  case OVERLOAD_FD_EXHAUSTED:
566  SET_TO_START_OF_HOUR(overload_stats.overload_fd_exhausted_time);
567  overload_stats.overload_fd_exhausted++;
568  break;
569  }
570 }
571 
572 /** Note down that we've reached a TCP port exhaustion. This triggers an
573  * overload general event. */
574 void
576 {
578  rep_hist_note_overload(OVERLOAD_GENERAL);
579 }
580 
581 /** Return the total number of TCP exhaustion times we've reached. */
582 uint64_t
584 {
585  return stats_n_tcp_exhaustion;
586 }
587 
588 /** Return the or_history_t for the OR with identity digest <b>id</b>,
589  * creating it if necessary. */
590 static or_history_t *
591 get_or_history(const char* id)
592 {
593  or_history_t *hist;
594 
595  if (tor_digest_is_zero(id))
596  return NULL;
597 
598  hist = digestmap_get(history_map, id);
599  if (!hist) {
600  hist = tor_malloc_zero(sizeof(or_history_t));
603  hist->since = hist->changed = time(NULL);
605  digestmap_set(history_map, id, hist);
606  }
607  return hist;
608 }
609 
610 /** Helper: free storage held by a single OR history entry. */
611 static void
612 free_or_history(void *_hist)
613 {
614  or_history_t *hist = _hist;
617  tor_free(hist);
618 }
619 
620 /** Initialize the static data structures for tracking history. */
621 void
623 {
624  history_map = digestmap_new();
625 }
626 
627 /** We have just decided that this router with identity digest <b>id</b> is
628  * reachable, meaning we will give it a "Running" flag for the next while. */
629 void
630 rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
631  const uint16_t at_port, time_t when)
632 {
633  or_history_t *hist = get_or_history(id);
634  int was_in_run = 1;
635  char tbuf[ISO_TIME_LEN+1];
636  int addr_changed, port_changed;
637 
638  tor_assert(hist);
639  tor_assert((!at_addr && !at_port) || (at_addr && at_port));
640 
641  addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) &&
642  tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0;
643  port_changed = at_port && hist->last_reached_port &&
644  at_port != hist->last_reached_port;
645 
646  if (!started_tracking_stability)
647  started_tracking_stability = time(NULL);
648  if (!hist->start_of_run) {
649  hist->start_of_run = when;
650  was_in_run = 0;
651  }
652  if (hist->start_of_downtime) {
653  long down_length;
654 
655  format_local_iso_time(tbuf, hist->start_of_downtime);
656  log_info(LD_HIST, "Router %s is now Running; it had been down since %s.",
657  hex_str(id, DIGEST_LEN), tbuf);
658  if (was_in_run)
659  log_info(LD_HIST, " (Paradoxically, it was already Running too.)");
660 
661  down_length = when - hist->start_of_downtime;
662  hist->total_weighted_time += down_length;
663  hist->start_of_downtime = 0;
664  } else if (addr_changed || port_changed) {
665  /* If we're reachable, but the address changed, treat this as some
666  * downtime. */
667  int penalty = get_options()->TestingTorNetwork ? 240 : 3600;
668  networkstatus_t *ns;
669 
670  if ((ns = networkstatus_get_latest_consensus())) {
671  int fresh_interval = (int)(ns->fresh_until - ns->valid_after);
672  int live_interval = (int)(ns->valid_until - ns->valid_after);
673  /* on average, a descriptor addr change takes .5 intervals to make it
674  * into a consensus, and half a liveness period to make it to
675  * clients. */
676  penalty = (int)(fresh_interval + live_interval) / 2;
677  }
678  format_local_iso_time(tbuf, hist->start_of_run);
679  log_info(LD_HIST,"Router %s still seems Running, but its address appears "
680  "to have changed since the last time it was reachable. I'm "
681  "going to treat it as having been down for %d seconds",
682  hex_str(id, DIGEST_LEN), penalty);
683  rep_hist_note_router_unreachable(id, when-penalty);
684  rep_hist_note_router_reachable(id, NULL, 0, when);
685  } else {
686  format_local_iso_time(tbuf, hist->start_of_run);
687  if (was_in_run)
688  log_debug(LD_HIST, "Router %s is still Running; it has been Running "
689  "since %s", hex_str(id, DIGEST_LEN), tbuf);
690  else
691  log_info(LD_HIST,"Router %s is now Running; it was previously untracked",
692  hex_str(id, DIGEST_LEN));
693  }
694  if (at_addr)
695  tor_addr_copy(&hist->last_reached_addr, at_addr);
696  if (at_port)
697  hist->last_reached_port = at_port;
698 }
699 
700 /** We have just decided that this router is unreachable, meaning
701  * we are taking away its "Running" flag. */
702 void
703 rep_hist_note_router_unreachable(const char *id, time_t when)
704 {
705  or_history_t *hist = get_or_history(id);
706  char tbuf[ISO_TIME_LEN+1];
707  int was_running = 0;
708  if (!started_tracking_stability)
709  started_tracking_stability = time(NULL);
710 
711  tor_assert(hist);
712  if (hist->start_of_run) {
713  /*XXXX We could treat failed connections differently from failed
714  * connect attempts. */
715  long run_length = when - hist->start_of_run;
716  format_local_iso_time(tbuf, hist->start_of_run);
717 
718  hist->total_run_weights += 1.0;
719  hist->start_of_run = 0;
720  if (run_length < 0) {
721  unsigned long penalty = -run_length;
722 #define SUBTRACT_CLAMPED(var, penalty) \
723  do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)
724 
725  SUBTRACT_CLAMPED(hist->weighted_run_length, penalty);
726  SUBTRACT_CLAMPED(hist->weighted_uptime, penalty);
727  } else {
728  hist->weighted_run_length += run_length;
729  hist->weighted_uptime += run_length;
730  hist->total_weighted_time += run_length;
731  }
732  was_running = 1;
733  log_info(LD_HIST, "Router %s is now non-Running: it had previously been "
734  "Running since %s. Its total weighted uptime is %lu/%lu.",
735  hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime,
736  hist->total_weighted_time);
737  }
738  if (!hist->start_of_downtime) {
739  hist->start_of_downtime = when;
740 
741  if (!was_running)
742  log_info(LD_HIST, "Router %s is now non-Running; it was previously "
743  "untracked.", hex_str(id, DIGEST_LEN));
744  } else {
745  if (!was_running) {
746  format_local_iso_time(tbuf, hist->start_of_downtime);
747 
748  log_info(LD_HIST, "Router %s is still non-Running; it has been "
749  "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf);
750  }
751  }
752 }
753 
754 /** Mark a router with ID <b>id</b> as non-Running, and retroactively declare
755  * that it has never been running: give it no stability and no WFU. */
756 void
757 rep_hist_make_router_pessimal(const char *id, time_t when)
758 {
759  or_history_t *hist = get_or_history(id);
760  tor_assert(hist);
761 
763 
764  hist->weighted_run_length = 0;
765  hist->weighted_uptime = 0;
766 }
767 
768 /** Helper: Discount all old MTBF data, if it is time to do so. Return
769  * the time at which we should next discount MTBF data. */
770 time_t
772 {
773  digestmap_iter_t *orhist_it;
774  const char *digest1;
775  or_history_t *hist;
776  void *hist_p;
777  double alpha = 1.0;
778 
779  if (!history_map)
780  history_map = digestmap_new();
785 
786  /* Okay, we should downrate the data. By how much? */
789  alpha *= STABILITY_ALPHA;
790  }
791 
792  log_info(LD_HIST, "Discounting all old stability info by a factor of %f",
793  alpha);
794 
795  /* Multiply every w_r_l, t_r_w pair by alpha. */
796  for (orhist_it = digestmap_iter_init(history_map);
797  !digestmap_iter_done(orhist_it);
798  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
799  digestmap_iter_get(orhist_it, &digest1, &hist_p);
800  hist = hist_p;
801 
802  hist->weighted_run_length =
803  (unsigned long)(hist->weighted_run_length * alpha);
804  hist->total_run_weights *= alpha;
805 
806  hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha);
807  hist->total_weighted_time = (unsigned long)
808  (hist->total_weighted_time * alpha);
809  }
810 
812 }
813 
814 /** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */
815 static double
816 get_stability(or_history_t *hist, time_t when)
817 {
818  long total = hist->weighted_run_length;
819  double total_weights = hist->total_run_weights;
820 
821  if (hist->start_of_run) {
822  /* We're currently in a run. Let total and total_weights hold the values
823  * they would hold if the current run were to end now. */
824  total += (when-hist->start_of_run);
825  total_weights += 1.0;
826  }
827  if (total_weights < STABILITY_EPSILON) {
828  /* Round down to zero, and avoid divide-by-zero. */
829  return 0.0;
830  }
831 
832  return total / total_weights;
833 }
834 
835 /** Return the total amount of time we've been observing, with each run of
836  * time downrated by the appropriate factor. */
837 static long
839 {
840  long total = hist->total_weighted_time;
841  if (hist->start_of_run) {
842  total += (when - hist->start_of_run);
843  } else if (hist->start_of_downtime) {
844  total += (when - hist->start_of_downtime);
845  }
846  return total;
847 }
848 
849 /** Helper: Return the weighted percent-of-time-online of the router with
850  * history <b>hist</b>. */
851 static double
853 {
854  long total = hist->total_weighted_time;
855  long up = hist->weighted_uptime;
856 
857  if (hist->start_of_run) {
858  long run_length = (when - hist->start_of_run);
859  up += run_length;
860  total += run_length;
861  } else if (hist->start_of_downtime) {
862  total += (when - hist->start_of_downtime);
863  }
864 
865  if (!total) {
866  /* Avoid calling anybody's uptime infinity (which should be impossible if
867  * the code is working), or NaN (which can happen for any router we haven't
868  * observed up or down yet). */
869  return 0.0;
870  }
871 
872  return ((double) up) / total;
873 }
874 
875 /** Return how long the router whose identity digest is <b>id</b> has
876  * been reachable. Return 0 if the router is unknown or currently deemed
877  * unreachable. */
878 long
879 rep_hist_get_uptime(const char *id, time_t when)
880 {
881  or_history_t *hist = get_or_history(id);
882  if (!hist)
883  return 0;
884  if (!hist->start_of_run || when < hist->start_of_run)
885  return 0;
886  return when - hist->start_of_run;
887 }
888 
889 /** Return an estimated MTBF for the router whose identity digest is
890  * <b>id</b>. Return 0 if the router is unknown. */
891 double
892 rep_hist_get_stability(const char *id, time_t when)
893 {
894  or_history_t *hist = get_or_history(id);
895  if (!hist)
896  return 0.0;
897 
898  return get_stability(hist, when);
899 }
900 
901 /** Return an estimated percent-of-time-online for the router whose identity
902  * digest is <b>id</b>. Return 0 if the router is unknown. */
903 double
904 rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
905 {
906  or_history_t *hist = get_or_history(id);
907  if (!hist)
908  return 0.0;
909 
910  return get_weighted_fractional_uptime(hist, when);
911 }
912 
913 /** Return a number representing how long we've known about the router whose
914  * digest is <b>id</b>. Return 0 if the router is unknown.
915  *
916  * Be careful: this measure increases monotonically as we know the router for
917  * longer and longer, but it doesn't increase linearly.
918  */
919 long
920 rep_hist_get_weighted_time_known(const char *id, time_t when)
921 {
922  or_history_t *hist = get_or_history(id);
923  if (!hist)
924  return 0;
925 
926  return get_total_weighted_time(hist, when);
927 }
928 
929 /** Return true if we've been measuring MTBFs for long enough to
930  * pronounce on Stability. */
931 int
933 {
934  /* XXXX++ This doesn't do so well when we change our opinion
935  * as to whether we're tracking router stability. */
936  return started_tracking_stability < time(NULL) - 4*60*60;
937 }
938 
939 /** Log all the reliability data we have remembered, with the chosen
940  * severity.
941  */
942 void
943 rep_hist_dump_stats(time_t now, int severity)
944 {
945  digestmap_iter_t *orhist_it;
946  const char *name1, *digest1;
947  char hexdigest1[HEX_DIGEST_LEN+1];
948  or_history_t *or_history;
949  void *or_history_p;
950  const node_t *node;
951 
952  rep_history_clean(now - get_options()->RephistTrackTime);
953 
954  tor_log(severity, LD_HIST, "--------------- Dumping history information:");
955 
956  for (orhist_it = digestmap_iter_init(history_map);
957  !digestmap_iter_done(orhist_it);
958  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
959  double s;
960  long stability;
961  digestmap_iter_get(orhist_it, &digest1, &or_history_p);
962  or_history = (or_history_t*) or_history_p;
963 
964  if ((node = node_get_by_id(digest1)) && node_get_nickname(node))
965  name1 = node_get_nickname(node);
966  else
967  name1 = "(unknown)";
968  base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
969  s = get_stability(or_history, now);
970  stability = (long)s;
971  tor_log(severity, LD_HIST,
972  "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
973  name1, hexdigest1,
974  stability/3600, (stability/60)%60, stability%60);
975  }
976 }
977 
978 /** Remove history info for routers/links that haven't changed since
979  * <b>before</b>.
980  */
981 void
982 rep_history_clean(time_t before)
983 {
984  int authority = authdir_mode(get_options());
985  or_history_t *or_history;
986  void *or_history_p;
987  digestmap_iter_t *orhist_it;
988  const char *d1;
989 
990  orhist_it = digestmap_iter_init(history_map);
991  while (!digestmap_iter_done(orhist_it)) {
992  int should_remove;
993  digestmap_iter_get(orhist_it, &d1, &or_history_p);
994  or_history = or_history_p;
995 
996  should_remove = authority ?
997  (or_history->total_run_weights < STABILITY_EPSILON &&
998  !or_history->start_of_run)
999  : (or_history->changed < before);
1000  if (should_remove) {
1001  orhist_it = digestmap_iter_next_rmv(history_map, orhist_it);
1002  free_or_history(or_history);
1003  continue;
1004  }
1005  orhist_it = digestmap_iter_next(history_map, orhist_it);
1006  }
1007 }
1008 
1009 /** Write MTBF data to disk. Return 0 on success, negative on failure.
1010  *
1011  * If <b>missing_means_down</b>, then if we're about to write an entry
1012  * that is still considered up but isn't in our routerlist, consider it
1013  * to be down. */
1014 int
1015 rep_hist_record_mtbf_data(time_t now, int missing_means_down)
1016 {
1017  char time_buf[ISO_TIME_LEN+1];
1018 
1019  digestmap_iter_t *orhist_it;
1020  const char *digest;
1021  void *or_history_p;
1022  or_history_t *hist;
1023  open_file_t *open_file = NULL;
1024  FILE *f;
1025 
1026  {
1027  char *filename = get_datadir_fname("router-stability");
1028  f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600,
1029  &open_file);
1030  tor_free(filename);
1031  if (!f)
1032  return -1;
1033  }
1034 
1035  /* File format is:
1036  * FormatLine *KeywordLine Data
1037  *
1038  * FormatLine = "format 1" NL
1039  * KeywordLine = Keyword SP Arguments NL
1040  * Data = "data" NL *RouterMTBFLine "." NL
1041  * RouterMTBFLine = Fingerprint SP WeightedRunLen SP
1042  * TotalRunWeights [SP S=StartRunTime] NL
1043  */
1044 #define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END
1045 #define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END
1046 
1047  PUT("format 2\n");
1048 
1049  format_iso_time(time_buf, time(NULL));
1050  PRINTF((f, "stored-at %s\n", time_buf));
1051 
1052  if (started_tracking_stability) {
1053  format_iso_time(time_buf, started_tracking_stability);
1054  PRINTF((f, "tracked-since %s\n", time_buf));
1055  }
1058  PRINTF((f, "last-downrated %s\n", time_buf));
1059  }
1060 
1061  PUT("data\n");
1062 
1063  /* XXX Nick: now bridge auths record this for all routers too.
1064  * Should we make them record it only for bridge routers? -RD
1065  * Not for 0.2.0. -NM */
1066  for (orhist_it = digestmap_iter_init(history_map);
1067  !digestmap_iter_done(orhist_it);
1068  orhist_it = digestmap_iter_next(history_map,orhist_it)) {
1069  char dbuf[HEX_DIGEST_LEN+1];
1070  const char *t = NULL;
1071  digestmap_iter_get(orhist_it, &digest, &or_history_p);
1072  hist = (or_history_t*) or_history_p;
1073 
1074  base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
1075 
1076  if (missing_means_down && hist->start_of_run &&
1078  /* We think this relay is running, but it's not listed in our
1079  * consensus. Somehow it fell out without telling us it went
1080  * down. Complain and also correct it. */
1081  log_info(LD_HIST,
1082  "Relay '%s' is listed as up in rephist, but it's not in "
1083  "our routerlist. Correcting.", dbuf);
1084  rep_hist_note_router_unreachable(digest, now);
1085  }
1086 
1087  PRINTF((f, "R %s\n", dbuf));
1088  if (hist->start_of_run > 0) {
1089  format_iso_time(time_buf, hist->start_of_run);
1090  t = time_buf;
1091  }
1092  PRINTF((f, "+MTBF %lu %.5f%s%s\n",
1094  t ? " S=" : "", t ? t : ""));
1095  t = NULL;
1096  if (hist->start_of_downtime > 0) {
1097  format_iso_time(time_buf, hist->start_of_downtime);
1098  t = time_buf;
1099  }
1100  PRINTF((f, "+WFU %lu %lu%s%s\n",
1101  hist->weighted_uptime, hist->total_weighted_time,
1102  t ? " S=" : "", t ? t : ""));
1103  }
1104 
1105  PUT(".\n");
1106 
1107 #undef PUT
1108 #undef PRINTF
1109 
1110  return finish_writing_to_file(open_file);
1111  err:
1112  abort_writing_to_file(open_file);
1113  return -1;
1114 }
1115 
1116 /** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and
1117  * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no
1118  * such line exists. */
1119 static int
1120 find_next_with(smartlist_t *sl, int i, const char *prefix)
1121 {
1122  for ( ; i < smartlist_len(sl); ++i) {
1123  const char *line = smartlist_get(sl, i);
1124  if (!strcmpstart(line, prefix))
1125  return i;
1126  if (!strcmpstart(line, "R "))
1127  return -1;
1128  }
1129  return -1;
1130 }
1131 
1132 /** How many bad times has parse_possibly_bad_iso_time() parsed? */
1133 static int n_bogus_times = 0;
1134 /** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but
1135  * round any pre-1970 date to Jan 1, 1970. */
1136 static int
1137 parse_possibly_bad_iso_time(const char *s, time_t *time_out)
1138 {
1139  int year;
1140  char b[5];
1141  strlcpy(b, s, sizeof(b));
1142  b[4] = '\0';
1143  year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL);
1144  if (year < 1970) {
1145  *time_out = 0;
1146  ++n_bogus_times;
1147  return 0;
1148  } else
1149  return parse_iso_time(s, time_out);
1150 }
1151 
1152 /** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which
1153  * says we started measuring at <b>started_measuring</b>. Return a new number
1154  * that's about as much before <b>now</b> as <b>t</b> was before
1155  * <b>stored_at</b>.
1156  */
1157 static inline time_t
1158 correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
1159 {
1160  if (t < started_measuring - 24*60*60*365)
1161  return 0;
1162  else if (t < started_measuring)
1163  return started_measuring;
1164  else if (t > stored_at)
1165  return 0;
1166  else {
1167  long run_length = stored_at - t;
1168  t = (time_t)(now - run_length);
1169  if (t < started_measuring)
1170  t = started_measuring;
1171  return t;
1172  }
1173 }
1174 
1175 /** Load MTBF data from disk. Returns 0 on success or recoverable error, -1
1176  * on failure. */
1177 int
1179 {
1180  /* XXXX won't handle being called while history is already populated. */
1181  smartlist_t *lines;
1182  const char *line = NULL;
1183  int r=0, i;
1184  time_t last_downrated = 0, stored_at = 0, tracked_since = 0;
1185  time_t latest_possible_start = now;
1186  long format = -1;
1187 
1188  {
1189  char *filename = get_datadir_fname("router-stability");
1190  char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
1191  tor_free(filename);
1192  if (!d)
1193  return -1;
1194  lines = smartlist_new();
1195  smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0);
1196  tor_free(d);
1197  }
1198 
1199  {
1200  const char *firstline;
1201  if (smartlist_len(lines)>4) {
1202  firstline = smartlist_get(lines, 0);
1203  if (!strcmpstart(firstline, "format "))
1204  format = tor_parse_long(firstline+strlen("format "),
1205  10, -1, LONG_MAX, NULL, NULL);
1206  }
1207  }
1208  if (format != 1 && format != 2) {
1209  log_warn(LD_HIST,
1210  "Unrecognized format in mtbf history file. Skipping.");
1211  goto err;
1212  }
1213  for (i = 1; i < smartlist_len(lines); ++i) {
1214  line = smartlist_get(lines, i);
1215  if (!strcmp(line, "data"))
1216  break;
1217  if (!strcmpstart(line, "last-downrated ")) {
1218  if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0)
1219  log_warn(LD_HIST,"Couldn't parse downrate time in mtbf "
1220  "history file.");
1221  }
1222  if (!strcmpstart(line, "stored-at ")) {
1223  if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0)
1224  log_warn(LD_HIST,"Couldn't parse stored time in mtbf "
1225  "history file.");
1226  }
1227  if (!strcmpstart(line, "tracked-since ")) {
1228  if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0)
1229  log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf "
1230  "history file.");
1231  }
1232  }
1233  if (last_downrated > now)
1234  last_downrated = now;
1235  if (tracked_since > now)
1236  tracked_since = now;
1237 
1238  if (!stored_at) {
1239  log_warn(LD_HIST, "No stored time recorded.");
1240  goto err;
1241  }
1242 
1243  if (line && !strcmp(line, "data"))
1244  ++i;
1245 
1246  n_bogus_times = 0;
1247 
1248  for (; i < smartlist_len(lines); ++i) {
1249  char digest[DIGEST_LEN];
1250  char hexbuf[HEX_DIGEST_LEN+1];
1251  char mtbf_timebuf[ISO_TIME_LEN+1];
1252  char wfu_timebuf[ISO_TIME_LEN+1];
1253  time_t start_of_run = 0;
1254  time_t start_of_downtime = 0;
1255  int have_mtbf = 0, have_wfu = 0;
1256  long wrl = 0;
1257  double trw = 0;
1258  long wt_uptime = 0, total_wt_time = 0;
1259  int n;
1260  or_history_t *hist;
1261  line = smartlist_get(lines, i);
1262  if (!strcmp(line, "."))
1263  break;
1264 
1265  mtbf_timebuf[0] = '\0';
1266  wfu_timebuf[0] = '\0';
1267 
1268  if (format == 1) {
1269  n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s",
1270  hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
1271  if (n != 3 && n != 5) {
1272  log_warn(LD_HIST, "Couldn't scan line %s", escaped(line));
1273  continue;
1274  }
1275  have_mtbf = 1;
1276  } else {
1277  // format == 2.
1278  int mtbf_idx, wfu_idx;
1279  if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN)
1280  continue;
1281  strlcpy(hexbuf, line+2, sizeof(hexbuf));
1282  mtbf_idx = find_next_with(lines, i+1, "+MTBF ");
1283  wfu_idx = find_next_with(lines, i+1, "+WFU ");
1284  if (mtbf_idx >= 0) {
1285  const char *mtbfline = smartlist_get(lines, mtbf_idx);
1286  n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s",
1287  &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
1288  if (n == 2 || n == 4) {
1289  have_mtbf = 1;
1290  } else {
1291  log_warn(LD_HIST, "Couldn't scan +MTBF line %s",
1292  escaped(mtbfline));
1293  }
1294  }
1295  if (wfu_idx >= 0) {
1296  const char *wfuline = smartlist_get(lines, wfu_idx);
1297  n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s",
1298  &wt_uptime, &total_wt_time,
1299  wfu_timebuf, wfu_timebuf+11);
1300  if (n == 2 || n == 4) {
1301  have_wfu = 1;
1302  } else {
1303  log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline));
1304  }
1305  }
1306  if (wfu_idx > i)
1307  i = wfu_idx;
1308  if (mtbf_idx > i)
1309  i = mtbf_idx;
1310  }
1311  if (base16_decode(digest, DIGEST_LEN,
1312  hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) {
1313  log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf));
1314  continue;
1315  }
1316  hist = get_or_history(digest);
1317  if (!hist)
1318  continue;
1319 
1320  if (have_mtbf) {
1321  if (mtbf_timebuf[0]) {
1322  mtbf_timebuf[10] = ' ';
1323  if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0)
1324  log_warn(LD_HIST, "Couldn't parse time %s",
1325  escaped(mtbf_timebuf));
1326  }
1327  hist->start_of_run = correct_time(start_of_run, now, stored_at,
1328  tracked_since);
1329  if (hist->start_of_run < latest_possible_start + wrl)
1330  latest_possible_start = (time_t)(hist->start_of_run - wrl);
1331 
1332  hist->weighted_run_length = wrl;
1333  hist->total_run_weights = trw;
1334  }
1335  if (have_wfu) {
1336  if (wfu_timebuf[0]) {
1337  wfu_timebuf[10] = ' ';
1338  if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0)
1339  log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf));
1340  }
1341  }
1342  hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at,
1343  tracked_since);
1344  hist->weighted_uptime = wt_uptime;
1345  hist->total_weighted_time = total_wt_time;
1346  }
1347  if (strcmp(line, "."))
1348  log_warn(LD_HIST, "Truncated MTBF file.");
1349 
1350  if (tracked_since < 86400*365) /* Recover from insanely early value. */
1351  tracked_since = latest_possible_start;
1352 
1353  stability_last_downrated = last_downrated;
1354  started_tracking_stability = tracked_since;
1355 
1356  goto done;
1357  err:
1358  r = -1;
1359  done:
1360  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
1361  smartlist_free(lines);
1362  return r;
1363 }
1364 
1365 /*** Exit port statistics ***/
1366 
1367 /* Some constants */
1368 /** To what multiple should byte numbers be rounded up? */
1369 #define EXIT_STATS_ROUND_UP_BYTES 1024
1370 /** To what multiple should stream counts be rounded up? */
1371 #define EXIT_STATS_ROUND_UP_STREAMS 4
1372 /** Number of TCP ports */
1373 #define EXIT_STATS_NUM_PORTS 65536
1374 /** Top n ports that will be included in exit stats. */
1375 #define EXIT_STATS_TOP_N_PORTS 10
1376 
1377 /* The following data structures are arrays and no fancy smartlists or maps,
1378  * so that all write operations can be done in constant time. This comes at
1379  * the price of some memory (1.25 MB) and linear complexity when writing
1380  * stats for measuring relays. */
1381 /** Number of bytes read in current period by exit port */
1382 static uint64_t *exit_bytes_read = NULL;
1383 /** Number of bytes written in current period by exit port */
1384 static uint64_t *exit_bytes_written = NULL;
1385 /** Number of streams opened in current period by exit port */
1386 static uint32_t *exit_streams = NULL;
1387 
1388 /** Start time of exit stats or 0 if we're not collecting exit stats. */
1390 
1391 /** Initialize exit port stats. */
1392 void
1394 {
1396  exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
1397  exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
1398  exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t));
1399 }
1400 
1401 /** Reset counters for exit port statistics. */
1402 void
1404 {
1406  memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1407  memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
1408  memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t));
1409 }
1410 
1411 /** Stop collecting exit port stats in a way that we can re-start doing
1412  * so in rep_hist_exit_stats_init(). */
1413 void
1415 {
1420 }
1421 
1422 /** Helper for qsort: compare two ints. Does not handle overflow properly,
1423  * but works fine for sorting an array of port numbers, which is what we use
1424  * it for. */
1425 static int
1426 compare_int_(const void *x, const void *y)
1427 {
1428  return (*(int*)x - *(int*)y);
1429 }
1430 
1431 /** Return a newly allocated string containing the exit port statistics
1432  * until <b>now</b>, or NULL if we're not collecting exit stats. Caller
1433  * must ensure start_of_exit_stats_interval is in the past. */
1434 char *
1436 {
1437  int i, j, top_elements = 0, cur_min_idx = 0, cur_port;
1438  uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS];
1439  int top_ports[EXIT_STATS_TOP_N_PORTS];
1440  uint64_t cur_bytes = 0, other_read = 0, other_written = 0,
1441  total_read = 0, total_written = 0;
1442  uint32_t total_streams = 0, other_streams = 0;
1443  smartlist_t *written_strings, *read_strings, *streams_strings;
1444  char *written_string, *read_string, *streams_string;
1445  char t[ISO_TIME_LEN+1];
1446  char *result;
1447 
1449  return NULL; /* Not initialized. */
1450 
1452 
1453  /* Go through all ports to find the n ports that saw most written and
1454  * read bytes.
1455  *
1456  * Invariant: at the end of the loop for iteration i,
1457  * total_read is the sum of all exit_bytes_read[0..i]
1458  * total_written is the sum of all exit_bytes_written[0..i]
1459  * total_stream is the sum of all exit_streams[0..i]
1460  *
1461  * top_elements = MAX(EXIT_STATS_TOP_N_PORTS,
1462  * #{j | 0 <= j <= i && volume(i) > 0})
1463  *
1464  * For all 0 <= j < top_elements,
1465  * top_bytes[j] > 0
1466  * 0 <= top_ports[j] <= 65535
1467  * top_bytes[j] = volume(top_ports[j])
1468  *
1469  * There is no j in 0..i and k in 0..top_elements such that:
1470  * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements]
1471  *
1472  * There is no j!=cur_min_idx in 0..top_elements such that:
1473  * top_bytes[j] < top_bytes[cur_min_idx]
1474  *
1475  * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x]
1476  *
1477  * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS)
1478  */
1479  for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
1480  total_read += exit_bytes_read[i];
1481  total_written += exit_bytes_written[i];
1482  total_streams += exit_streams[i];
1483  cur_bytes = exit_bytes_read[i] + exit_bytes_written[i];
1484  if (cur_bytes == 0) {
1485  continue;
1486  }
1487  if (top_elements < EXIT_STATS_TOP_N_PORTS) {
1488  top_bytes[top_elements] = cur_bytes;
1489  top_ports[top_elements++] = i;
1490  } else if (cur_bytes > top_bytes[cur_min_idx]) {
1491  top_bytes[cur_min_idx] = cur_bytes;
1492  top_ports[cur_min_idx] = i;
1493  } else {
1494  continue;
1495  }
1496  cur_min_idx = 0;
1497  for (j = 1; j < top_elements; j++) {
1498  if (top_bytes[j] < top_bytes[cur_min_idx]) {
1499  cur_min_idx = j;
1500  }
1501  }
1502  }
1503 
1504  /* Add observations of top ports to smartlists. */
1505  written_strings = smartlist_new();
1506  read_strings = smartlist_new();
1507  streams_strings = smartlist_new();
1508  other_read = total_read;
1509  other_written = total_written;
1510  other_streams = total_streams;
1511  /* Sort the ports; this puts them out of sync with top_bytes, but we
1512  * won't be using top_bytes again anyway */
1513  qsort(top_ports, top_elements, sizeof(int), compare_int_);
1514  for (j = 0; j < top_elements; j++) {
1515  cur_port = top_ports[j];
1516  if (exit_bytes_written[cur_port] > 0) {
1517  uint64_t num = round_uint64_to_next_multiple_of(
1518  exit_bytes_written[cur_port],
1520  num /= 1024;
1521  smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
1522  cur_port, (num));
1523  other_written -= exit_bytes_written[cur_port];
1524  }
1525  if (exit_bytes_read[cur_port] > 0) {
1526  uint64_t num = round_uint64_to_next_multiple_of(
1527  exit_bytes_read[cur_port],
1529  num /= 1024;
1530  smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
1531  cur_port, (num));
1532  other_read -= exit_bytes_read[cur_port];
1533  }
1534  if (exit_streams[cur_port] > 0) {
1535  uint32_t num = round_uint32_to_next_multiple_of(
1536  exit_streams[cur_port],
1538  smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num);
1539  other_streams -= exit_streams[cur_port];
1540  }
1541  }
1542 
1543  /* Add observations of other ports in a single element. */
1544  other_written = round_uint64_to_next_multiple_of(other_written,
1546  other_written /= 1024;
1547  smartlist_add_asprintf(written_strings, "other=%"PRIu64,
1548  (other_written));
1549  other_read = round_uint64_to_next_multiple_of(other_read,
1551  other_read /= 1024;
1552  smartlist_add_asprintf(read_strings, "other=%"PRIu64,
1553  (other_read));
1554  other_streams = round_uint32_to_next_multiple_of(other_streams,
1556  smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
1557 
1558  /* Join all observations in single strings. */
1559  written_string = smartlist_join_strings(written_strings, ",", 0, NULL);
1560  read_string = smartlist_join_strings(read_strings, ",", 0, NULL);
1561  streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL);
1562  SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp));
1563  SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp));
1564  SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp));
1565  smartlist_free(written_strings);
1566  smartlist_free(read_strings);
1567  smartlist_free(streams_strings);
1568 
1569  /* Put everything together. */
1570  format_iso_time(t, now);
1571  tor_asprintf(&result, "exit-stats-end %s (%d s)\n"
1572  "exit-kibibytes-written %s\n"
1573  "exit-kibibytes-read %s\n"
1574  "exit-streams-opened %s\n",
1575  t, (unsigned) (now - start_of_exit_stats_interval),
1576  written_string,
1577  read_string,
1578  streams_string);
1579  tor_free(written_string);
1580  tor_free(read_string);
1581  tor_free(streams_string);
1582  return result;
1583 }
1584 
1585 /** If 24 hours have passed since the beginning of the current exit port
1586  * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly
1587  * overwriting an existing file) and reset counters. Return when we would
1588  * next want to write exit stats or 0 if we never want to write. */
1589 time_t
1591 {
1592  char *str = NULL;
1593 
1595  return 0; /* Not initialized. */
1596  if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now)
1597  goto done; /* Not ready to write. */
1598 
1599  log_info(LD_HIST, "Writing exit port statistics to disk.");
1600 
1601  /* Generate history string. */
1602  str = rep_hist_format_exit_stats(now);
1603 
1604  /* Reset counters. */
1606 
1607  /* Try to write to disk. */
1608  if (!check_or_create_data_subdir("stats")) {
1609  write_to_data_subdir("stats", "exit-stats", str, "exit port statistics");
1610  }
1611 
1612  done:
1613  tor_free(str);
1614  return start_of_exit_stats_interval + WRITE_STATS_INTERVAL;
1615 }
1616 
1617 /** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b>
1618  * bytes to/from an exit connection to <b>port</b>. */
1619 void
1620 rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
1621  size_t num_read)
1622 {
1624  return; /* Not initialized. */
1625  exit_bytes_written[port] += num_written;
1626  exit_bytes_read[port] += num_read;
1627  log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an "
1628  "exit connection to port %d.",
1629  (unsigned long)num_written, (unsigned long)num_read, port);
1630 }
1631 
1632 /** Note that we opened an exit stream to <b>port</b>. */
1633 void
1635 {
1637  return; /* Not initialized. */
1638  exit_streams[port]++;
1639  log_debug(LD_HIST, "Opened exit stream to port %d", port);
1640 }
1641 
1642 /*** Exit streams statistics ***/
1643 
1644 /** Number of BEGIN streams seen. */
1645 static uint64_t streams_begin_seen;
1646 /** Number of BEGIN_DIR streams seen. */
1647 static uint64_t streams_begindir_seen;
1648 /** Number of RESOLVE streams seen. */
1649 static uint64_t streams_resolve_seen;
1650 
1651 /** Note a stream as seen for the given relay command. */
1652 void
1653 rep_hist_note_exit_stream(unsigned int cmd)
1654 {
1655  switch (cmd) {
1656  case RELAY_COMMAND_BEGIN:
1658  break;
1659  case RELAY_COMMAND_BEGIN_DIR:
1661  break;
1662  case RELAY_COMMAND_RESOLVE:
1664  break;
1665  default:
1666  tor_assert_nonfatal_unreached_once();
1667  break;
1668  }
1669 }
1670 
1671 /** Return number of stream seen for the given command. */
1672 uint64_t
1674 {
1675  switch (cmd) {
1676  case RELAY_COMMAND_BEGIN:
1677  return streams_begin_seen;
1678  case RELAY_COMMAND_BEGIN_DIR:
1679  return streams_begindir_seen;
1680  case RELAY_COMMAND_RESOLVE:
1681  return streams_resolve_seen;
1682  default:
1683  return 0;
1684  }
1685 }
1686 
1687 /******* Connections statistics *******/
1688 
1689 #define CONN_DIRECTION_INITIATED 0
1690 #define CONN_DIRECTION_RECEIVED 1
1691 
1692 #define CONN_DIRECTION(from_listener) \
1693  (from_listener) ? CONN_DIRECTION_RECEIVED : CONN_DIRECTION_INITIATED
1694 
1695 /** Number of connections created as in seen per direction per type. */
1696 static uint64_t conn_num_created_v4[2][CONN_TYPE_MAX_];
1697 static uint64_t conn_num_created_v6[2][CONN_TYPE_MAX_];
1698 /** Number of connections opened per direction per type. */
1699 static uint64_t conn_num_opened_v4[2][CONN_TYPE_MAX_];
1700 static uint64_t conn_num_opened_v6[2][CONN_TYPE_MAX_];
1701 /** Number of connections rejected per type. Always inbound. */
1702 static uint64_t conn_num_rejected_v4[CONN_TYPE_MAX_];
1703 static uint64_t conn_num_rejected_v6[CONN_TYPE_MAX_];
1704 
1705 /** Note that a connection has opened of the given type. */
1706 void
1707 rep_hist_note_conn_opened(bool from_listener, unsigned int type, int af)
1708 {
1709  tor_assert(type <= CONN_TYPE_MAX_);
1710 
1711  unsigned int dir = CONN_DIRECTION(from_listener);
1712 
1713  switch (af) {
1714  case AF_INET:
1715  conn_num_created_v4[dir][type]++;
1716  conn_num_opened_v4[dir][type]++;
1717  break;
1718  case AF_INET6:
1719  conn_num_created_v6[dir][type]++;
1720  conn_num_opened_v6[dir][type]++;
1721  break;
1722  default:
1723  /* Ignore non IP connections at this point in time. */
1724  break;
1725  }
1726 }
1727 
1728 /** Note that a connection has closed of the given type. */
1729 void
1730 rep_hist_note_conn_closed(bool from_listener, unsigned int type, int af)
1731 {
1732  tor_assert(type <= CONN_TYPE_MAX_);
1733 
1734  unsigned int dir = CONN_DIRECTION(from_listener);
1735 
1736  switch (af) {
1737  case AF_INET:
1738  if (conn_num_opened_v4[dir][type] > 0) {
1739  conn_num_opened_v4[dir][type]--;
1740  }
1741  break;
1742  case AF_INET6:
1743  if (conn_num_opened_v6[dir][type] > 0) {
1744  conn_num_opened_v6[dir][type]--;
1745  }
1746  break;
1747  default:
1748  /* Ignore non IP connections at this point in time. */
1749  break;
1750  }
1751 }
1752 
1753 /** Note that a connection has rejected of the given type. */
1754 void
1755 rep_hist_note_conn_rejected(unsigned int type, int af)
1756 {
1757  tor_assert(type <= CONN_TYPE_MAX_);
1758 
1759  switch (af) {
1760  case AF_INET:
1761  conn_num_rejected_v4[type]++;
1762  break;
1763  case AF_INET6:
1764  conn_num_rejected_v6[type]++;
1765  break;
1766  default:
1767  /* Ignore non IP connections at this point in time. */
1768  break;
1769  }
1770 }
1771 
1772 /** Return number of created connections of the given type. */
1773 uint64_t
1774 rep_hist_get_conn_created(bool from_listener, unsigned int type, int af)
1775 {
1776  tor_assert(type <= CONN_TYPE_MAX_);
1777  unsigned int dir = CONN_DIRECTION(from_listener);
1778  switch (af) {
1779  case AF_INET:
1780  return conn_num_created_v4[dir][type];
1781  case AF_INET6:
1782  return conn_num_created_v6[dir][type];
1783  default:
1784  return 0;
1785  }
1786 }
1787 
1788 /** Return number of opened connections of the given type. */
1789 uint64_t
1790 rep_hist_get_conn_opened(bool from_listener, unsigned int type, int af)
1791 {
1792  tor_assert(type <= CONN_TYPE_MAX_);
1793  unsigned int dir = CONN_DIRECTION(from_listener);
1794  switch (af) {
1795  case AF_INET:
1796  return conn_num_opened_v4[dir][type];
1797  case AF_INET6:
1798  return conn_num_opened_v6[dir][type];
1799  default:
1800  return 0;
1801  }
1802 }
1803 
1804 /** Return number of opened connections of the given type. */
1805 uint64_t
1806 rep_hist_get_conn_rejected(unsigned int type, int af)
1807 {
1808  tor_assert(type <= CONN_TYPE_MAX_);
1809  switch (af) {
1810  case AF_INET:
1811  return conn_num_rejected_v4[type];
1812  case AF_INET6:
1813  return conn_num_rejected_v6[type];
1814  default:
1815  return 0;
1816  }
1817 }
1818 
1819 /*** cell statistics ***/
1820 
1821 /** Start of the current buffer stats interval or 0 if we're not
1822  * collecting buffer statistics. */
1824 
1825 /** Initialize buffer stats. */
1826 void
1828 {
1830 }
1831 
1832 /** Statistics from a single circuit. Collected when the circuit closes, or
1833  * when we flush statistics to disk. */
1834 typedef struct circ_buffer_stats_t {
1835  /** Average number of cells in the circuit's queue */
1837  /** Average time a cell waits in the queue. */
1839  /** Total number of cells sent over this circuit */
1842 
1843 /** List of circ_buffer_stats_t. */
1845 
1846 /** Remember cell statistics <b>mean_num_cells_in_queue</b>,
1847  * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a
1848  * circuit. */
1849 void
1850 rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
1851  double mean_time_cells_in_queue, uint32_t processed_cells)
1852 {
1853  circ_buffer_stats_t *stats;
1855  return; /* Not initialized. */
1856  stats = tor_malloc_zero(sizeof(circ_buffer_stats_t));
1857  stats->mean_num_cells_in_queue = mean_num_cells_in_queue;
1858  stats->mean_time_cells_in_queue = mean_time_cells_in_queue;
1859  stats->processed_cells = processed_cells;
1863 }
1864 
1865 /** Remember cell statistics for circuit <b>circ</b> at time
1866  * <b>end_of_interval</b> and reset cell counters in case the circuit
1867  * remains open in the next measurement interval. */
1868 void
1869 rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
1870 {
1871  time_t start_of_interval;
1872  int interval_length;
1873  or_circuit_t *orcirc;
1874  double mean_num_cells_in_queue, mean_time_cells_in_queue;
1875  uint32_t processed_cells;
1876  if (CIRCUIT_IS_ORIGIN(circ))
1877  return;
1878  orcirc = TO_OR_CIRCUIT(circ);
1879  if (!orcirc->processed_cells)
1880  return;
1881  start_of_interval = (circ->timestamp_created.tv_sec >
1883  (time_t)circ->timestamp_created.tv_sec :
1885  interval_length = (int) (end_of_interval - start_of_interval);
1886  if (interval_length <= 0)
1887  return;
1888  processed_cells = orcirc->processed_cells;
1889  /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */
1890  mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time /
1891  (double) interval_length / 1000.0 / 2.0;
1892  mean_time_cells_in_queue =
1893  (double) orcirc->total_cell_waiting_time /
1894  (double) orcirc->processed_cells;
1895  orcirc->total_cell_waiting_time = 0;
1896  orcirc->processed_cells = 0;
1897  rep_hist_add_buffer_stats(mean_num_cells_in_queue,
1898  mean_time_cells_in_queue,
1899  processed_cells);
1900 }
1901 
1902 /** Sorting helper: return -1, 1, or 0 based on comparison of two
1903  * circ_buffer_stats_t */
1904 static int
1905 buffer_stats_compare_entries_(const void **_a, const void **_b)
1906 {
1907  const circ_buffer_stats_t *a = *_a, *b = *_b;
1908  if (a->processed_cells < b->processed_cells)
1909  return 1;
1910  else if (a->processed_cells > b->processed_cells)
1911  return -1;
1912  else
1913  return 0;
1914 }
1915 
1916 /** Stop collecting cell stats in a way that we can re-start doing so in
1917  * rep_hist_buffer_stats_init(). */
1918 void
1920 {
1922 }
1923 
1924 /** Clear history of circuit statistics and set the measurement interval
1925  * start to <b>now</b>. */
1926 void
1928 {
1932  stats, tor_free(stats));
1935 }
1936 
1937 /** Return a newly allocated string containing the buffer statistics until
1938  * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must
1939  * ensure start_of_buffer_stats_interval is in the past. */
1940 char *
1942 {
1943 #define SHARES 10
1944  uint64_t processed_cells[SHARES];
1945  uint32_t circs_in_share[SHARES];
1946  int number_of_circuits, i;
1947  double queued_cells[SHARES], time_in_queue[SHARES];
1948  smartlist_t *processed_cells_strings, *queued_cells_strings,
1949  *time_in_queue_strings;
1950  char *processed_cells_string, *queued_cells_string,
1951  *time_in_queue_string;
1952  char t[ISO_TIME_LEN+1];
1953  char *result;
1954 
1956  return NULL; /* Not initialized. */
1957 
1959 
1960  /* Calculate deciles if we saw at least one circuit. */
1961  memset(processed_cells, 0, SHARES * sizeof(uint64_t));
1962  memset(circs_in_share, 0, SHARES * sizeof(uint32_t));
1963  memset(queued_cells, 0, SHARES * sizeof(double));
1964  memset(time_in_queue, 0, SHARES * sizeof(double));
1967  number_of_circuits = smartlist_len(circuits_for_buffer_stats);
1968  if (number_of_circuits > 0) {
1971  i = 0;
1973  circ_buffer_stats_t *, stats)
1974  {
1975  int share = i++ * SHARES / number_of_circuits;
1976  processed_cells[share] += stats->processed_cells;
1977  queued_cells[share] += stats->mean_num_cells_in_queue;
1978  time_in_queue[share] += stats->mean_time_cells_in_queue;
1979  circs_in_share[share]++;
1980  }
1981  SMARTLIST_FOREACH_END(stats);
1982  }
1983 
1984  /* Write deciles to strings. */
1985  processed_cells_strings = smartlist_new();
1986  queued_cells_strings = smartlist_new();
1987  time_in_queue_strings = smartlist_new();
1988  for (i = 0; i < SHARES; i++) {
1989  smartlist_add_asprintf(processed_cells_strings,
1990  "%"PRIu64, !circs_in_share[i] ? 0 :
1991  (processed_cells[i] /
1992  circs_in_share[i]));
1993  }
1994  for (i = 0; i < SHARES; i++) {
1995  smartlist_add_asprintf(queued_cells_strings, "%.2f",
1996  circs_in_share[i] == 0 ? 0.0 :
1997  queued_cells[i] / (double) circs_in_share[i]);
1998  }
1999  for (i = 0; i < SHARES; i++) {
2000  smartlist_add_asprintf(time_in_queue_strings, "%.0f",
2001  circs_in_share[i] == 0 ? 0.0 :
2002  time_in_queue[i] / (double) circs_in_share[i]);
2003  }
2004 
2005  /* Join all observations in single strings. */
2006  processed_cells_string = smartlist_join_strings(processed_cells_strings,
2007  ",", 0, NULL);
2008  queued_cells_string = smartlist_join_strings(queued_cells_strings,
2009  ",", 0, NULL);
2010  time_in_queue_string = smartlist_join_strings(time_in_queue_strings,
2011  ",", 0, NULL);
2012  SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp));
2013  SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp));
2014  SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp));
2015  smartlist_free(processed_cells_strings);
2016  smartlist_free(queued_cells_strings);
2017  smartlist_free(time_in_queue_strings);
2018 
2019  /* Put everything together. */
2020  format_iso_time(t, now);
2021  tor_asprintf(&result, "cell-stats-end %s (%d s)\n"
2022  "cell-processed-cells %s\n"
2023  "cell-queued-cells %s\n"
2024  "cell-time-in-queue %s\n"
2025  "cell-circuits-per-decile %d\n",
2026  t, (unsigned) (now - start_of_buffer_stats_interval),
2027  processed_cells_string,
2028  queued_cells_string,
2029  time_in_queue_string,
2030  CEIL_DIV(number_of_circuits, SHARES));
2031  tor_free(processed_cells_string);
2032  tor_free(queued_cells_string);
2033  tor_free(time_in_queue_string);
2034  return result;
2035 #undef SHARES
2036 }
2037 
2038 /** If 24 hours have passed since the beginning of the current buffer
2039  * stats period, write buffer stats to $DATADIR/stats/buffer-stats
2040  * (possibly overwriting an existing file) and reset counters. Return
2041  * when we would next want to write buffer stats or 0 if we never want to
2042  * write. */
2043 time_t
2045 {
2046  char *str = NULL;
2047 
2049  return 0; /* Not initialized. */
2050  if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now)
2051  goto done; /* Not ready to write */
2052 
2053  /* Add open circuits to the history. */
2055  rep_hist_buffer_stats_add_circ(circ, now);
2056  }
2057  SMARTLIST_FOREACH_END(circ);
2058 
2059  /* Generate history string. */
2060  str = rep_hist_format_buffer_stats(now);
2061 
2062  /* Reset both buffer history and counters of open circuits. */
2064 
2065  /* Try to write to disk. */
2066  if (!check_or_create_data_subdir("stats")) {
2067  write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics");
2068  }
2069 
2070  done:
2071  tor_free(str);
2072  return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL;
2073 }
2074 
2075 /*** Descriptor serving statistics ***/
2076 
2077 /** Digestmap to track which descriptors were downloaded this stats
2078  * collection interval. It maps descriptor digest to pointers to 1,
2079  * effectively turning this into a list. */
2080 static digestmap_t *served_descs = NULL;
2081 
2082 /** Number of how many descriptors were downloaded in total during this
2083  * interval. */
2084 static unsigned long total_descriptor_downloads;
2085 
2086 /** Start time of served descs stats or 0 if we're not collecting those. */
2088 
2089 /** Initialize descriptor stats. */
2090 void
2092 {
2093  if (served_descs) {
2094  log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were "
2095  "already initialized. This is probably harmless.");
2096  return; // Already initialized
2097  }
2098  served_descs = digestmap_new();
2101 }
2102 
2103 /** Reset served descs stats to empty, starting a new interval <b>now</b>. */
2104 static void
2106 {
2109 }
2110 
2111 /** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is
2112  * safe to be called again. */
2113 void
2115 {
2116  digestmap_free(served_descs, NULL);
2117  served_descs = NULL;
2120 }
2121 
2122 /** Helper for rep_hist_desc_stats_write(). Return a newly allocated string
2123  * containing the served desc statistics until now, or NULL if we're not
2124  * collecting served desc stats. Caller must ensure that now is not before
2125  * start_of_served_descs_stats_interval. */
2126 static char *
2128 {
2129  char t[ISO_TIME_LEN+1];
2130  char *result;
2131 
2132  digestmap_iter_t *iter;
2133  const char *key;
2134  void *val;
2135  unsigned size;
2136  int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0;
2137  int n = 0;
2138 
2140  return NULL;
2141 
2142  size = digestmap_size(served_descs);
2143  if (size > 0) {
2144  vals = tor_calloc(size, sizeof(int));
2145  for (iter = digestmap_iter_init(served_descs);
2146  !digestmap_iter_done(iter);
2147  iter = digestmap_iter_next(served_descs, iter)) {
2148  uintptr_t count;
2149  digestmap_iter_get(iter, &key, &val);
2150  count = (uintptr_t)val;
2151  vals[n++] = (int)count;
2152  (void)key;
2153  }
2154  max = find_nth_int(vals, size, size-1);
2155  q3 = find_nth_int(vals, size, (3*size-1)/4);
2156  md = find_nth_int(vals, size, (size-1)/2);
2157  q1 = find_nth_int(vals, size, (size-1)/4);
2158  min = find_nth_int(vals, size, 0);
2159  tor_free(vals);
2160  }
2161 
2162  format_iso_time(t, now);
2163 
2164  tor_asprintf(&result,
2165  "served-descs-stats-end %s (%d s) total=%lu unique=%u "
2166  "max=%d q3=%d md=%d q1=%d min=%d\n",
2167  t,
2168  (unsigned) (now - start_of_served_descs_stats_interval),
2170  size, max, q3, md, q1, min);
2171 
2172  return result;
2173 }
2174 
2175 /** If WRITE_STATS_INTERVAL seconds have passed since the beginning of
2176  * the current served desc stats interval, write the stats to
2177  * $DATADIR/stats/served-desc-stats (possibly appending to an existing file)
2178  * and reset the state for the next interval. Return when we would next want
2179  * to write served desc stats or 0 if we won't want to write. */
2180 time_t
2182 {
2183  char *filename = NULL, *str = NULL;
2184 
2186  return 0; /* We're not collecting stats. */
2187  if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now)
2188  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
2189 
2190  str = rep_hist_format_desc_stats(now);
2191  tor_assert(str != NULL);
2192 
2193  if (check_or_create_data_subdir("stats") < 0) {
2194  goto done;
2195  }
2196  filename = get_datadir_fname2("stats", "served-desc-stats");
2197  if (append_bytes_to_file(filename, str, strlen(str), 0) < 0)
2198  log_warn(LD_HIST, "Unable to write served descs statistics to disk!");
2199 
2201 
2202  done:
2203  tor_free(filename);
2204  tor_free(str);
2205  return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
2206 }
2207 
2208 /** Called to note that we've served a given descriptor (by
2209  * digest). Increments the count of descriptors served, and the number
2210  * of times we've served this descriptor. */
2211 void
2212 rep_hist_note_desc_served(const char * desc)
2213 {
2214  void *val;
2215  uintptr_t count;
2216  if (!served_descs)
2217  return; // We're not collecting stats
2218  val = digestmap_get(served_descs, desc);
2219  count = (uintptr_t)val;
2220  if (count != INT_MAX)
2221  ++count;
2222  digestmap_set(served_descs, desc, (void*)count);
2224 }
2225 
2226 /*** Connection statistics ***/
2227 
2228 /** Internal statistics to track how many requests of each type of
2229  * handshake we've received, and how many we've assigned to cpuworkers.
2230  * Useful for seeing trends in cpu load.
2231  *
2232  * They are reset at every heartbeat.
2233  * @{ */
2235 STATIC int onion_handshakes_assigned[MAX_ONION_STAT_TYPE+1] = {0};
2236 /**@}*/
2237 
2238 /** Counters keeping the same stats as above but for the entire duration of the
2239  * process (not reset). */
2241 static uint64_t stats_n_onionskin_dropped[MAX_ONION_STAT_TYPE+1] = {0};
2242 
2243 /* We use a scale here so we can represent percentages with decimal points by
2244  * scaling the value by this factor and so 0.5% becomes a value of 500.
2245  * Default is 1% and thus min and max range is 0 to 100%. */
2246 #define OVERLOAD_ONIONSKIN_NTOR_PERCENT_SCALE 1000.0
2247 #define OVERLOAD_ONIONSKIN_NTOR_PERCENT_DEFAULT 1000
2248 #define OVERLOAD_ONIONSKIN_NTOR_PERCENT_MIN 0
2249 #define OVERLOAD_ONIONSKIN_NTOR_PERCENT_MAX 100000
2250 
2251 /** Consensus parameter: indicate what fraction of ntor onionskin drop over the
2252  * total number of requests must be reached before we trigger a general
2253  * overload signal.*/
2255  OVERLOAD_ONIONSKIN_NTOR_PERCENT_DEFAULT /
2256  OVERLOAD_ONIONSKIN_NTOR_PERCENT_SCALE / 100.0;
2257 
2258 /* Number of seconds for the assessment period. Default is 6 hours (21600) and
2259  * the min max range is within a 32bit value. We align this period to the
2260  * Heartbeat so the logs would match this period more or less. */
2261 #define OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_DEFAULT (60 * 60 * 6)
2262 #define OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_MIN 0
2263 #define OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_MAX INT32_MAX
2264 
2265 /** Consensus parameter: Period, in seconds, over which we count the number of
2266  * ntor onionskins requests and how many were dropped. After that period, we
2267  * assess if we trigger an overload or not. */
2269  OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_DEFAULT;
2270 
2271 /** Structure containing information for an assessment period of the onionskin
2272  * drop overload general signal.
2273  *
2274  * It is used to track, within a time period, how many requests we've gotten
2275  * and how many were dropped. The overload general signal is decided from these
2276  * depending on some consensus parameters. */
2277 typedef struct {
2278  /** Total number of ntor onionskin requested for an assessment period. */
2280 
2281  /** Total number of dropped ntor onionskins for an assessment period. */
2282  uint64_t n_ntor_dropped;
2283 
2284  /** When is the next assessment time of the general overload for ntor
2285  * onionskin drop. Once this time is reached, all stats are reset and this
2286  * time is set to the next assessment time. */
2289 
2290 /** Keep track of the onionskin requests for an assessment period. */
2292 
2293 /**
2294  * We combine ntorv3 and ntor into the same stat, so we must
2295  * use this function to convert the cell type to a stat index.
2296  */
2297 static inline uint16_t
2299 {
2300  if (type == ONION_HANDSHAKE_TYPE_NTOR_V3) {
2301  return ONION_HANDSHAKE_TYPE_NTOR;
2302  }
2303 
2304  if (BUG(type > MAX_ONION_STAT_TYPE)) {
2305  return MAX_ONION_STAT_TYPE; // use ntor if out of range
2306  }
2307 
2308  return type;
2309 }
2310 
2311 /** Assess our ntor handshake statistics and decide if we need to emit a
2312  * general overload signal.
2313  *
2314  * Regardless of overloaded or not, if the assessment time period has passed,
2315  * the stats are reset back to 0 and the assessment time period updated.
2316  *
2317  * This is called when a ntor handshake is _requested_ because we want to avoid
2318  * to have an asymmetric situation where requested counter is reset to 0 but
2319  * then a drop happens leading to the drop counter being incremented while the
2320  * requested counter is 0. */
2321 static void
2323 {
2324  /* Initialize the time. Should be done once. */
2326  goto reset;
2327  }
2328 
2329  /* Not the time yet. */
2331  goto done;
2332  }
2333 
2334  /* Make sure we have enough requests to be able to make a proper assessment.
2335  * We want to avoid 1 single request/drop to trigger an overload as we want
2336  * at least the number of requests to be above the scale of our fraction. */
2338  OVERLOAD_ONIONSKIN_NTOR_PERCENT_SCALE) {
2339  goto done;
2340  }
2341 
2342  /* Lets see if we can signal a general overload. */
2343  double fraction = (double) overload_onionskin_assessment.n_ntor_dropped /
2345  if (fraction >= overload_onionskin_ntor_fraction) {
2346  log_notice(LD_HIST, "General overload -> Ntor dropped (%" PRIu64 ") "
2347  "fraction %.4f%% is above threshold of %.4f%%",
2349  fraction * 100.0,
2351  rep_hist_note_overload(OVERLOAD_GENERAL);
2352  }
2353 
2354  reset:
2355  /* Reset counters for the next period. */
2360 
2361  done:
2362  return;
2363 }
2364 
2365 /** A new onionskin (using the <b>type</b> handshake) has arrived. */
2366 void
2368 {
2369  uint16_t stat = onionskin_type_to_stat(type);
2370 
2372 
2373  /* Only relays get to record requested onionskins. */
2374  if (stat == ONION_HANDSHAKE_TYPE_NTOR) {
2375  /* Assess if we've reached the overload general signal. */
2377 
2379  }
2380 }
2381 
2382 /** We've sent an onionskin (using the <b>type</b> handshake) to a
2383  * cpuworker. */
2384 void
2386 {
2387  onion_handshakes_assigned[onionskin_type_to_stat(type)]++;
2389 }
2390 
2391 /** We've just drop an onionskin (using the <b>type</b> handshake) due to being
2392  * overloaded. */
2393 void
2395 {
2396  uint16_t stat = onionskin_type_to_stat(type);
2397 
2398  stats_n_onionskin_dropped[stat]++;
2399 
2400  /* Only relays get to record requested onionskins. */
2401  if (stat == ONION_HANDSHAKE_TYPE_NTOR) {
2402  /* Note the dropped ntor in the overload assessment object. */
2404  }
2405 }
2406 
2407 /** Get the circuit handshake value that is requested. */
2408 MOCK_IMPL(int,
2410 {
2412 }
2413 
2414 /** Get the circuit handshake value that is assigned. */
2415 MOCK_IMPL(int,
2417 {
2418  return onion_handshakes_assigned[onionskin_type_to_stat(type)];
2419 }
2420 
2421 /** Get the total number of circuit handshake value that is assigned. */
2422 MOCK_IMPL(uint64_t,
2424 {
2426 }
2427 
2428 /** Get the total number of circuit handshake value that is dropped. */
2429 MOCK_IMPL(uint64_t,
2431 {
2432  return stats_n_onionskin_dropped[onionskin_type_to_stat(type)];
2433 }
2434 
2435 /** Log our onionskin statistics since the last time we were called. */
2436 void
2438 {
2439  (void)now;
2440  log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: "
2441  "%d/%d TAP, %d/%d NTor.",
2442  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP],
2443  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP],
2444  onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR],
2445  onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]);
2446  memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned));
2448 }
2449 
2450 /* Hidden service statistics section */
2451 
2452 /** Start of the current hidden service stats interval or 0 if we're
2453  * not collecting hidden service statistics. */
2455 
2456 /** Our v2 statistics structure singleton. */
2457 static hs_v2_stats_t *hs_v2_stats = NULL;
2458 
2459 /** HSv2 stats */
2460 
2461 /** Allocate, initialize and return an hs_v2_stats_t structure. */
2462 static hs_v2_stats_t *
2464 {
2465  hs_v2_stats_t *new_hs_v2_stats = tor_malloc_zero(sizeof(hs_v2_stats_t));
2466 
2467  return new_hs_v2_stats;
2468 }
2469 
2470 #define hs_v2_stats_free(val) \
2471  FREE_AND_NULL(hs_v2_stats_t, hs_v2_stats_free_, (val))
2472 
2473 /** Free an hs_v2_stats_t structure. */
2474 static void
2475 hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats)
2476 {
2477  if (!victim_hs_v2_stats) {
2478  return;
2479  }
2480  tor_free(victim_hs_v2_stats);
2481 }
2482 
2483 /** Clear history of hidden service statistics and set the measurement
2484  * interval start to <b>now</b>. */
2485 static void
2487 {
2488  if (!hs_v2_stats) {
2490  }
2491 
2492  hs_v2_stats->rp_v2_relay_cells_seen = 0;
2493 
2495 }
2496 
2497 /*** HSv3 stats ******/
2498 
2499 /** Start of the current hidden service stats interval or 0 if we're not
2500  * collecting hidden service statistics.
2501  *
2502  * This is particularly important for v3 statistics since this variable
2503  * controls the start time of initial v3 stats collection. It's initialized by
2504  * rep_hist_hs_stats_init() to the next time period start (i.e. 12:00UTC), and
2505  * should_collect_v3_stats() ensures that functions that collect v3 stats do
2506  * not do so sooner than that.
2507  *
2508  * Collecting stats from 12:00UTC to 12:00UTC is extremely important for v3
2509  * stats because rep_hist_hsdir_stored_maybe_new_v3_onion() uses the blinded
2510  * key of each onion service as its double-counting index. Onion services
2511  * rotate their descriptor at around 00:00UTC which means that their blinded
2512  * key also changes around that time. However the precise time that onion
2513  * services rotate their descriptors is actually when they fetch a new
2514  * 00:00UTC consensus and that happens at a random time (e.g. it can even
2515  * happen at 02:00UTC). This means that if we started keeping v3 stats at
2516  * around 00:00UTC we wouldn't be able to tell when onion services change
2517  * their blinded key and hence we would double count an unpredictable amount
2518  * of them (for example, if an onion service fetches the 00:00UTC consensus at
2519  * 01:00UTC it would upload to its old HSDir at 00:45UTC, and then to a
2520  * different HSDir at 01:50UTC).
2521  *
2522  * For this reason, we start collecting statistics at 12:00UTC. This way we
2523  * know that by the time we stop collecting statistics for that time period 24
2524  * hours later, all the onion services have switched to their new blinded
2525  * key. This way we can predict much better how much double counting has been
2526  * performed.
2527  */
2529 
2530 /** Our v3 statistics structure singleton. */
2531 static hs_v3_stats_t *hs_v3_stats = NULL;
2532 
2533 /** Allocate, initialize and return an hs_v3_stats_t structure. */
2534 static hs_v3_stats_t *
2536 {
2537  hs_v3_stats_t *new_hs_v3_stats = tor_malloc_zero(sizeof(hs_v3_stats_t));
2538  new_hs_v3_stats->v3_onions_seen_this_period = digest256map_new();
2539 
2540  return new_hs_v3_stats;
2541 }
2542 
2543 #define hs_v3_stats_free(val) \
2544  FREE_AND_NULL(hs_v3_stats_t, hs_v3_stats_free_, (val))
2545 
2546 /** Free an hs_v3_stats_t structure. */
2547 static void
2548 hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
2549 {
2550  if (!victim_hs_v3_stats) {
2551  return;
2552  }
2553 
2554  digest256map_free(victim_hs_v3_stats->v3_onions_seen_this_period, NULL);
2555  tor_free(victim_hs_v3_stats);
2556 }
2557 
2558 /** Clear history of hidden service statistics and set the measurement
2559  * interval start to <b>now</b>. */
2560 static void
2562 {
2563  if (!hs_v3_stats) {
2565  }
2566 
2567  digest256map_free(hs_v3_stats->v3_onions_seen_this_period, NULL);
2568  hs_v3_stats->v3_onions_seen_this_period = digest256map_new();
2569 
2570  hs_v3_stats->rp_v3_relay_cells_seen = 0;
2571 
2573 }
2574 
2575 /** Return true if it's a good time to collect v3 stats.
2576  *
2577  * v3 stats have a strict stats collection period (from 12:00UTC to 12:00UTC
2578  * on the real network). We don't want to collect statistics if (for example)
2579  * we just booted and it's 03:00UTC; we will wait until 12:00UTC before we
2580  * start collecting statistics to make sure that the final result represents
2581  * the whole collection period. This behavior is controlled by
2582  * rep_hist_hs_stats_init().
2583  */
2584 MOCK_IMPL(STATIC bool,
2586 {
2588 }
2589 
2590 /** We just received a new descriptor with <b>blinded_key</b>. See if we've
2591  * seen this blinded key before, and if not add it to the stats. */
2592 void
2593 rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key)
2594 {
2595  /* Return early if we don't collect HSv3 stats, or if it's not yet the time
2596  * to collect them. */
2597  if (!hs_v3_stats || !should_collect_v3_stats()) {
2598  return;
2599  }
2600 
2601  bool seen_before =
2602  !!digest256map_get(hs_v3_stats->v3_onions_seen_this_period,
2603  blinded_key);
2604 
2605  log_info(LD_GENERAL, "Considering v3 descriptor with %s (%sseen before)",
2606  safe_str(hex_str((char*)blinded_key, 32)),
2607  seen_before ? "" : "not ");
2608 
2609  /* Count it if we haven't seen it before. */
2610  if (!seen_before) {
2611  digest256map_set(hs_v3_stats->v3_onions_seen_this_period,
2612  blinded_key, (void*)(uintptr_t)1);
2613  }
2614 }
2615 
2616 /** We saw a new HS relay cell: count it!
2617  * If <b>is_v2</b> is set then it's a v2 RP cell, otherwise it's a v3. */
2618 void
2620 {
2621  log_debug(LD_GENERAL, "New RP cell (%d)", is_v2);
2622 
2623  if (is_v2 && hs_v2_stats) {
2624  hs_v2_stats->rp_v2_relay_cells_seen++;
2625  } else if (!is_v2 && hs_v3_stats && should_collect_v3_stats()) {
2626  hs_v3_stats->rp_v3_relay_cells_seen++;
2627  }
2628 }
2629 
2630 /** Generic HS stats code */
2631 
2632 /** Initialize v2 and v3 hidden service statistics. */
2633 void
2635 {
2636  if (!hs_v2_stats) {
2638  }
2639 
2640  /* Start collecting v2 stats straight away */
2642 
2643  if (!hs_v3_stats) {
2645  }
2646 
2647  /* Start collecting v3 stats at the next 12:00 UTC */
2649 }
2650 
2651 /** Stop collecting hidden service stats in a way that we can re-start
2652  * doing so in rep_hist_buffer_stats_init(). */
2653 void
2655 {
2658 }
2659 
2660 /** Stats reporting code */
2661 
2662 /* The number of cells that are supposed to be hidden from the adversary
2663  * by adding noise from the Laplace distribution. This value, divided by
2664  * EPSILON, is Laplace parameter b. It must be greater than 0. */
2665 #define REND_CELLS_DELTA_F 2048
2666 /* Security parameter for obfuscating number of cells with a value between
2667  * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same
2668  * time make statistics less usable. */
2669 #define REND_CELLS_EPSILON 0.3
2670 /* The number of cells that are supposed to be hidden from the adversary
2671  * by rounding up to the next multiple of this number. */
2672 #define REND_CELLS_BIN_SIZE 1024
2673 /* The number of service identities that are supposed to be hidden from the
2674  * adversary by adding noise from the Laplace distribution. This value,
2675  * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */
2676 #define ONIONS_SEEN_DELTA_F 8
2677 /* Security parameter for obfuscating number of service identities with a
2678  * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but
2679  * at the same time make statistics less usable. */
2680 #define ONIONS_SEEN_EPSILON 0.3
2681 /* The number of service identities that are supposed to be hidden from
2682  * the adversary by rounding up to the next multiple of this number. */
2683 #define ONIONS_SEEN_BIN_SIZE 8
2684 
2685 /** Allocate and return a string containing hidden service stats that
2686  * are meant to be placed in the extra-info descriptor.
2687  *
2688  * Function works for both v2 and v3 stats depending on <b>is_v3</b>. */
2689 STATIC char *
2690 rep_hist_format_hs_stats(time_t now, bool is_v3)
2691 {
2692  char t[ISO_TIME_LEN+1];
2693  char *hs_stats_string;
2694  int64_t obfuscated_onions_seen, obfuscated_cells_seen;
2695 
2696  uint64_t rp_cells_seen = is_v3 ?
2697  hs_v3_stats->rp_v3_relay_cells_seen : hs_v2_stats->rp_v2_relay_cells_seen;
2698  size_t onions_seen = is_v3 ?
2699  digest256map_size(hs_v3_stats->v3_onions_seen_this_period) : 0;
2700  time_t start_of_hs_stats_interval = is_v3 ?
2702 
2703  uint64_t rounded_cells_seen
2704  = round_uint64_to_next_multiple_of(rp_cells_seen, REND_CELLS_BIN_SIZE);
2705  rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX);
2706  obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen,
2708  REND_CELLS_DELTA_F, REND_CELLS_EPSILON);
2709 
2710  uint64_t rounded_onions_seen =
2711  round_uint64_to_next_multiple_of(onions_seen, ONIONS_SEEN_BIN_SIZE);
2712  rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX);
2713  obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen,
2714  crypto_rand_double(), ONIONS_SEEN_DELTA_F,
2715  ONIONS_SEEN_EPSILON);
2716 
2717  format_iso_time(t, now);
2718  tor_asprintf(&hs_stats_string, "%s %s (%u s)\n"
2719  "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n"
2720  "%s %"PRId64" delta_f=%d epsilon=%.2f bin_size=%d\n",
2721  is_v3 ? "hidserv-v3-stats-end" : "hidserv-stats-end",
2722  t, (unsigned) (now - start_of_hs_stats_interval),
2723  is_v3 ?
2724  "hidserv-rend-v3-relayed-cells" : "hidserv-rend-relayed-cells",
2725  obfuscated_cells_seen, REND_CELLS_DELTA_F,
2726  REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
2727  is_v3 ? "hidserv-dir-v3-onions-seen" :"hidserv-dir-onions-seen",
2728  obfuscated_onions_seen, ONIONS_SEEN_DELTA_F,
2729  ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
2730 
2731  return hs_stats_string;
2732 }
2733 
2734 /** If 24 hours have passed since the beginning of the current HS
2735  * stats period, write buffer stats to $DATADIR/stats/hidserv-v3-stats
2736  * (possibly overwriting an existing file) and reset counters. Return
2737  * when we would next want to write buffer stats or 0 if we never want to
2738  * write. Function works for both v2 and v3 stats depending on <b>is_v3</b>.
2739  */
2740 time_t
2741 rep_hist_hs_stats_write(time_t now, bool is_v3)
2742 {
2743  char *str = NULL;
2744 
2745  time_t start_of_hs_stats_interval = is_v3 ?
2747 
2748  if (!start_of_hs_stats_interval) {
2749  return 0; /* Not initialized. */
2750  }
2751 
2752  if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) {
2753  goto done; /* Not ready to write */
2754  }
2755 
2756  /* Generate history string. */
2757  str = rep_hist_format_hs_stats(now, is_v3);
2758 
2759  /* Reset HS history. */
2760  if (is_v3) {
2762  } else {
2764  }
2765 
2766  /* Try to write to disk. */
2767  if (!check_or_create_data_subdir("stats")) {
2768  write_to_data_subdir("stats",
2769  is_v3 ? "hidserv-v3-stats" : "hidserv-stats",
2770  str, "hidden service stats");
2771  }
2772 
2773  done:
2774  tor_free(str);
2775  return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
2776 }
2777 
2778 static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
2779 
2780 /** Note that we negotiated link protocol version <b>link_proto</b>, on
2781  * a connection that started here iff <b>started_here</b> is true.
2782  */
2783 void
2784 rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
2785 {
2786  started_here = !!started_here; /* force to 0 or 1 */
2787  if (link_proto > MAX_LINK_PROTO) {
2788  log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
2789  return;
2790  }
2791 
2792  link_proto_count[link_proto][started_here]++;
2793 }
2794 
2795 /**
2796  * Update the maximum count of total pending channel padding timers
2797  * in this period.
2798  */
2799 void
2800 rep_hist_padding_count_timers(uint64_t num_timers)
2801 {
2802  if (num_timers > padding_current.maximum_chanpad_timers) {
2804  }
2805 }
2806 
2807 /**
2808  * Count a cell that we sent for padding overhead statistics.
2809  *
2810  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
2811  * counted for PADDING_TYPE_TOTAL.
2812  */
2813 void
2815 {
2816  switch (type) {
2817  case PADDING_TYPE_DROP:
2819  break;
2820  case PADDING_TYPE_CELL:
2822  break;
2823  case PADDING_TYPE_TOTAL:
2825  break;
2828  break;
2831  break;
2832  }
2833 }
2834 
2835 /**
2836  * Count a cell that we've received for padding overhead statistics.
2837  *
2838  * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
2839  * counted for PADDING_TYPE_TOTAL.
2840  */
2841 void
2843 {
2844  switch (type) {
2845  case PADDING_TYPE_DROP:
2847  break;
2848  case PADDING_TYPE_CELL:
2850  break;
2851  case PADDING_TYPE_TOTAL:
2853  break;
2856  break;
2859  break;
2860  }
2861 }
2862 
2863 /**
2864  * Reset our current padding statistics. Called once every 24 hours.
2865  */
2866 void
2868 {
2869  memset(&padding_current, 0, sizeof(padding_current));
2870 }
2871 
2872 /**
2873  * Copy our current cell counts into a structure for listing in our
2874  * extra-info descriptor. Also perform appropriate rounding and redaction.
2875  *
2876  * This function is called once every 24 hours.
2877  */
2878 #define MIN_CELL_COUNTS_TO_PUBLISH 1
2879 #define ROUND_CELL_COUNTS_TO 10000
2880 void
2881 rep_hist_prep_published_padding_counts(time_t now)
2882 {
2884 
2887  memset(&padding_published, 0, sizeof(padding_published));
2888  return;
2889  }
2890 
2892 #define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
2893  ROUND_CELL_COUNTS_TO)
2894  ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
2895  ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
2896  ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
2897  ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
2898  ROUND_AND_SET_COUNT(padding_published.write_cell_count);
2899  ROUND_AND_SET_COUNT(padding_published.read_cell_count);
2900  ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
2901  ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
2902  ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
2903  ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
2904 #undef ROUND_AND_SET_COUNT
2905 }
2906 
2907 /**
2908  * Returns an allocated string for extra-info documents for publishing
2909  * padding statistics from the last 24 hour interval.
2910  */
2911 char *
2913 {
2914  char *result = NULL;
2915 
2918  return NULL;
2919  }
2920 
2921  tor_asprintf(&result, "padding-counts %s (%d s)"
2922  " bin-size=%"PRIu64
2923  " write-drop=%"PRIu64
2924  " write-pad=%"PRIu64
2925  " write-total=%"PRIu64
2926  " read-drop=%"PRIu64
2927  " read-pad=%"PRIu64
2928  " read-total=%"PRIu64
2929  " enabled-read-pad=%"PRIu64
2930  " enabled-read-total=%"PRIu64
2931  " enabled-write-pad=%"PRIu64
2932  " enabled-write-total=%"PRIu64
2933  " max-chanpad-timers=%"PRIu64
2934  "\n",
2937  (uint64_t)ROUND_CELL_COUNTS_TO,
2949  );
2950 
2951  return result;
2952 }
2953 
2954 /** Log a heartbeat message explaining how many connections of each link
2955  * protocol version we have used.
2956  */
2957 void
2959 {
2960  smartlist_t *lines = smartlist_new();
2961 
2962  for (int i = 1; i <= MAX_LINK_PROTO; i++) {
2963  char *line = NULL;
2964  tor_asprintf(&line, "initiated %"PRIu64" and received "
2965  "%"PRIu64" v%d connections", link_proto_count[i][1],
2966  link_proto_count[i][0], i);
2967  smartlist_add(lines, line);
2968  }
2969 
2970  char *log_line = smartlist_join_strings(lines, "; ", 0, NULL);
2971 
2972  log_notice(LD_HEARTBEAT, "Since startup we %s.", log_line);
2973 
2974  SMARTLIST_FOREACH(lines, char *, s, tor_free(s));
2975  smartlist_free(lines);
2976  tor_free(log_line);
2977 }
2978 
2979 /** Free all storage held by the OR/link history caches, by the
2980  * bandwidth history arrays, by the port history, or by statistics . */
2981 void
2983 {
2984  hs_v2_stats_free(hs_v2_stats);
2985  hs_v3_stats_free(hs_v3_stats);
2986  digestmap_free(history_map, free_or_history);
2987 
2993 
2996  tor_free(s));
2997  smartlist_free(circuits_for_buffer_stats);
2999  }
3002 
3003  tor_assert_nonfatal(rephist_total_alloc == 0);
3004  tor_assert_nonfatal_once(rephist_total_num == 0);
3005 }
3006 
3007 /** Called just before the consensus will be replaced. Update the consensus
3008  * parameters in case they changed. */
3009 void
3011 {
3013  networkstatus_get_param(ns, "overload_onionskin_ntor_scale_percent",
3014  OVERLOAD_ONIONSKIN_NTOR_PERCENT_DEFAULT,
3015  OVERLOAD_ONIONSKIN_NTOR_PERCENT_MIN,
3016  OVERLOAD_ONIONSKIN_NTOR_PERCENT_MAX) /
3017  OVERLOAD_ONIONSKIN_NTOR_PERCENT_SCALE / 100.0;
3018 
3020  networkstatus_get_param(ns, "overload_onionskin_ntor_period_secs",
3021  OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_DEFAULT,
3022  OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_MIN,
3023  OVERLOAD_ONIONSKIN_NTOR_PERIOD_SECS_MAX);
3024 }
3025 
3026 #ifdef TOR_UNIT_TESTS
3027 /* only exists for unit tests: get HSv2 stats object */
3028 const hs_v2_stats_t *
3029 rep_hist_get_hs_v2_stats(void)
3030 {
3031  return hs_v2_stats;
3032 }
3033 
3034 /* only exists for unit tests: get HSv2 stats object */
3035 const hs_v3_stats_t *
3036 rep_hist_get_hs_v3_stats(void)
3037 {
3038  return hs_v3_stats;
3039 }
3040 #endif /* defined(TOR_UNIT_TESTS) */
void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src)
Definition: address.c:933
void tor_addr_make_unspec(tor_addr_t *a)
Definition: address.c:225
int tor_addr_compare(const tor_addr_t *addr1, const tor_addr_t *addr2, tor_addr_comparison_t how)
Definition: address.c:984
int tor_addr_is_null(const tor_addr_t *addr)
Definition: address.c:780
time_t approx_time(void)
Definition: approx_time.c:32
int authdir_mode(const or_options_t *options)
Definition: authmode.c:25
Header file for directory authority mode.
const char * hex_str(const char *from, size_t fromlen)
Definition: binascii.c:34
int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:506
void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen)
Definition: binascii.c:478
smartlist_t * circuit_get_global_list(void)
Definition: circuitlist.c:705
or_circuit_t * TO_OR_CIRCUIT(circuit_t *x)
Definition: circuitlist.c:165
Header file for circuitlist.c.
#define CIRCUIT_IS_ORIGIN(c)
Definition: circuitlist.h:147
const or_options_t * get_options(void)
Definition: config.c:926
int check_or_create_data_subdir(const char *subdir)
Definition: config.c:7126
int write_to_data_subdir(const char *subdir, const char *fname, const char *str, const char *descr)
Definition: config.c:7145
Header file for config.c.
Header file for connection.c.
int connection_or_digest_is_known_relay(const char *id_digest)
Header file for connection_or.c.
void conn_stats_free_all(void)
Definition: connstats.c:112
Header for feature/stats/connstats.c.
#define HEX_DIGEST_LEN
Definition: crypto_digest.h:35
Common functions for using (pseudo-)random number generators.
double crypto_rand_double(void)
#define DIGEST_LEN
Definition: digest_sizes.h:20
const char * escaped(const char *s)
Definition: escape.c:126
#define RFTS_IGNORE_MISSING
Definition: files.h:101
int finish_writing_to_file(open_file_t *file_data)
Definition: files.c:465
int append_bytes_to_file(const char *fname, const char *str, size_t len, int bin)
Definition: files.c:554
FILE * start_writing_to_stdio_file(const char *fname, int open_flags, int mode, open_file_t **data_out)
Definition: files.c:398
int abort_writing_to_file(open_file_t *file_data)
Definition: files.c:473
time_t hs_get_start_time_of_next_time_period(time_t now)
Definition: hs_common.c:324
int64_t add_laplace_noise(int64_t signal_, double random_, double delta_f, double epsilon)
Definition: laplace.c:51
Header for laplace.c.
void tor_log(int severity, log_domain_mask_t domain, const char *format,...)
Definition: log.c:590
#define LD_HEARTBEAT
Definition: log.h:103
#define LD_BUG
Definition: log.h:86
#define LD_HIST
Definition: log.h:99
#define LD_GENERAL
Definition: log.h:62
#define tor_free(p)
Definition: malloc.h:56
uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor)
Definition: muldiv.c:35
uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor)
Definition: muldiv.c:50
networkstatus_t * networkstatus_get_latest_consensus(void)
int32_t networkstatus_get_param(const networkstatus_t *ns, const char *param_name, int32_t default_val, int32_t min_val, int32_t max_val)
Header file for networkstatus.c.
Networkstatus consensus/vote structure.
const char * node_get_nickname(const node_t *node)
Definition: nodelist.c:1450
const node_t * node_get_by_id(const char *identity_digest)
Definition: nodelist.c:226
Header file for nodelist.c.
Master header file for Tor-specific functionality.
Header for order.c.
long tor_parse_long(const char *s, int base, long min, long max, int *ok, char **next)
Definition: parse_int.c:59
void predicted_ports_free_all(void)
Header file for predict_ports.c.
int tor_asprintf(char **strp, const char *fmt,...)
Definition: printf.c:75
void rep_hist_note_dns_error(int type, uint8_t error)
Definition: rephist.c:372
static dns_stats_t dns_all_stats
Definition: rephist.c:281
static int parse_possibly_bad_iso_time(const char *s, time_t *time_out)
Definition: rephist.c:1137
void rep_hist_note_desc_served(const char *desc)
Definition: rephist.c:2212
void rep_hist_consensus_has_changed(const networkstatus_t *ns)
Definition: rephist.c:3010
void rep_hist_buffer_stats_term(void)
Definition: rephist.c:1919
time_t rep_hist_downrate_old_runs(time_t now)
Definition: rephist.c:771
uint64_t rep_hist_get_conn_created(bool from_listener, unsigned int type, int af)
Definition: rephist.c:1774
void rep_hist_note_exit_bytes(uint16_t port, size_t num_written, size_t num_read)
Definition: rephist.c:1620
static time_t start_of_hs_v2_stats_interval
Definition: rephist.c:2454
void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
Definition: rephist.c:2784
static uint64_t stats_n_onionskin_assigned[MAX_ONION_STAT_TYPE+1]
Definition: rephist.c:2240
void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr, const uint16_t at_port, time_t when)
Definition: rephist.c:630
uint64_t rephist_total_alloc
Definition: rephist.c:95
void rep_hist_seen_new_rp_cell(bool is_v2)
Definition: rephist.c:2619
void rep_hist_note_conn_closed(bool from_listener, unsigned int type, int af)
Definition: rephist.c:1730
static padding_counts_t padding_current
Definition: rephist.c:173
void rep_hist_desc_stats_term(void)
Definition: rephist.c:2114
static hs_v3_stats_t * hs_v3_stats
Definition: rephist.c:2531
static time_t stability_last_downrated
Definition: rephist.c:181
static long get_total_weighted_time(or_history_t *hist, time_t when)
Definition: rephist.c:838
static digestmap_t * served_descs
Definition: rephist.c:2080
uint64_t rep_hist_get_n_dns_request(int type)
Definition: rephist.c:355
void rep_hist_hs_stats_term(void)
Definition: rephist.c:2654
static time_t start_of_served_descs_stats_interval
Definition: rephist.c:2087
static dns_stats_t * get_dns_stats_by_type(const int type)
Definition: rephist.c:286
void rep_hist_reset_padding_counts(void)
Definition: rephist.c:2867
double rep_hist_get_stability(const char *id, time_t when)
Definition: rephist.c:892
static bool overload_happened_recently(time_t overload_time, int n_hours)
Definition: rephist.c:444
static void hs_v2_stats_free_(hs_v2_stats_t *victim_hs_v2_stats)
Definition: rephist.c:2475
static uint32_t * exit_streams
Definition: rephist.c:1386
#define EXIT_STATS_TOP_N_PORTS
Definition: rephist.c:1375
int rep_hist_load_mtbf_data(time_t now)
Definition: rephist.c:1178
void rep_hist_note_circuit_handshake_dropped(uint16_t type)
Definition: rephist.c:2394
static or_history_t * get_or_history(const char *id)
Definition: rephist.c:591
void rep_hist_free_all(void)
Definition: rephist.c:2982
static uint64_t * exit_bytes_written
Definition: rephist.c:1384
STATIC int onion_handshakes_requested[MAX_ONION_STAT_TYPE+1]
Definition: rephist.c:2234
static overload_stats_t overload_stats
Definition: rephist.c:211
void rep_hist_init(void)
Definition: rephist.c:622
#define EXIT_STATS_ROUND_UP_BYTES
Definition: rephist.c:1369
int rep_hist_get_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:2416
time_t rep_hist_desc_stats_write(time_t now)
Definition: rephist.c:2181
char * rep_hist_get_overload_stats_lines(void)
Definition: rephist.c:491
void rep_hist_note_exit_stream(unsigned int cmd)
Definition: rephist.c:1653
void rep_hist_buffer_stats_init(time_t now)
Definition: rephist.c:1827
static uint64_t streams_begin_seen
Definition: rephist.c:1645
uint64_t rep_hist_get_conn_rejected(unsigned int type, int af)
Definition: rephist.c:1806
void rep_hist_exit_stats_init(time_t now)
Definition: rephist.c:1393
static void hs_v3_stats_free_(hs_v3_stats_t *victim_hs_v3_stats)
Definition: rephist.c:2548
void rep_hist_note_exit_stream_opened(uint16_t port)
Definition: rephist.c:1634
static char * rep_hist_format_desc_stats(time_t now)
Definition: rephist.c:2127
double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
Definition: rephist.c:904
static hs_v2_stats_t * hs_v2_stats_new(void)
Definition: rephist.c:2463
static uint64_t stats_n_read_limit_reached
Definition: rephist.c:215
void rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
Definition: rephist.c:1869
static int find_next_with(smartlist_t *sl, int i, const char *prefix)
Definition: rephist.c:1120
uint64_t rep_hist_get_circuit_n_handshake_assigned(uint16_t type)
Definition: rephist.c:2423
static double overload_onionskin_ntor_fraction
Definition: rephist.c:2254
void rep_hist_note_dns_request(int type)
Definition: rephist.c:431
void rep_hist_padding_count_write(padding_type_t type)
Definition: rephist.c:2814
char * rep_hist_get_overload_general_line(void)
Definition: rephist.c:473
long rep_hist_get_uptime(const char *id, time_t when)
Definition: rephist.c:879
static int buffer_stats_compare_entries_(const void **_a, const void **_b)
Definition: rephist.c:1905
static digestmap_t * history_map
Definition: rephist.c:187
long rep_hist_get_weighted_time_known(const char *id, time_t when)
Definition: rephist.c:920
static padding_counts_t padding_published
Definition: rephist.c:177
void rep_hist_note_conn_rejected(unsigned int type, int af)
Definition: rephist.c:1755
static uint64_t * exit_bytes_read
Definition: rephist.c:1382
void rep_hist_dump_stats(time_t now, int severity)
Definition: rephist.c:943
static time_t start_of_exit_stats_interval
Definition: rephist.c:1389
static uint16_t onionskin_type_to_stat(uint16_t type)
Definition: rephist.c:2298
static uint64_t stats_n_tcp_exhaustion
Definition: rephist.c:219
void rep_hist_reset_buffer_stats(time_t now)
Definition: rephist.c:1927
static time_t start_of_buffer_stats_interval
Definition: rephist.c:1823
void rep_hist_note_router_unreachable(const char *id, time_t when)
Definition: rephist.c:703
void rep_hist_hs_stats_init(time_t now)
Definition: rephist.c:2634
void rep_hist_hsdir_stored_maybe_new_v3_onion(const uint8_t *blinded_key)
Definition: rephist.c:2593
static int n_bogus_times
Definition: rephist.c:1133
static time_t start_of_hs_v3_stats_interval
Definition: rephist.c:2528
char * rep_hist_format_exit_stats(time_t now)
Definition: rephist.c:1435
static overload_dns_stats_t overload_dns_stats
Definition: rephist.c:244
static uint64_t conn_num_opened_v4[2][CONN_TYPE_MAX_]
Definition: rephist.c:1699
static overload_onionskin_assessment_t overload_onionskin_assessment
Definition: rephist.c:2291
void rep_hist_log_link_protocol_counts(void)
Definition: rephist.c:2958
STATIC char * rep_hist_format_hs_stats(time_t now, bool is_v3)
Definition: rephist.c:2690
uint32_t rephist_total_num
Definition: rephist.c:97
char * rep_hist_get_padding_count_lines(void)
Definition: rephist.c:2912
#define SET_TO_START_OF_HOUR(a)
Definition: rephist.c:532
static hs_v3_stats_t * hs_v3_stats_new(void)
Definition: rephist.c:2535
void rep_hist_add_buffer_stats(double mean_num_cells_in_queue, double mean_time_cells_in_queue, uint32_t processed_cells)
Definition: rephist.c:1850
static double get_weighted_fractional_uptime(or_history_t *hist, time_t when)
Definition: rephist.c:852
uint64_t rep_hist_get_conn_opened(bool from_listener, unsigned int type, int af)
Definition: rephist.c:1790
void rep_hist_note_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:2367
static uint64_t conn_num_rejected_v4[CONN_TYPE_MAX_]
Definition: rephist.c:1702
int rep_hist_get_circuit_handshake_requested(uint16_t type)
Definition: rephist.c:2409
void rep_hist_note_conn_opened(bool from_listener, unsigned int type, int af)
Definition: rephist.c:1707
void rep_history_clean(time_t before)
Definition: rephist.c:982
static hs_v2_stats_t * hs_v2_stats
Definition: rephist.c:2457
static uint64_t streams_resolve_seen
Definition: rephist.c:1649
void rep_hist_desc_stats_init(time_t now)
Definition: rephist.c:2091
uint64_t rep_hist_get_n_dns_error(int type, uint8_t error)
Definition: rephist.c:314
static void free_or_history(void *_hist)
Definition: rephist.c:612
#define EXIT_STATS_ROUND_UP_STREAMS
Definition: rephist.c:1371
#define STABILITY_ALPHA
Definition: rephist.c:104
uint64_t rep_hist_get_n_read_limit_reached(void)
Definition: rephist.c:458
static void rep_hist_reset_hs_v2_stats(time_t now)
Definition: rephist.c:2486
#define REND_CELLS_DELTA_F
Definition: rephist.c:2665
static unsigned long total_descriptor_downloads
Definition: rephist.c:2084
uint64_t rep_hist_get_n_tcp_exhaustion(void)
Definition: rephist.c:583
#define STABILITY_EPSILON
Definition: rephist.c:101
static smartlist_t * circuits_for_buffer_stats
Definition: rephist.c:1844
static int compare_int_(const void *x, const void *y)
Definition: rephist.c:1426
static time_t correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
Definition: rephist.c:1158
int rep_hist_record_mtbf_data(time_t now, int missing_means_down)
Definition: rephist.c:1015
char * rep_hist_format_buffer_stats(time_t now)
Definition: rephist.c:1941
static uint64_t streams_begindir_seen
Definition: rephist.c:1647
#define EXIT_STATS_NUM_PORTS
Definition: rephist.c:1373
void rep_hist_reset_exit_stats(time_t now)
Definition: rephist.c:1403
void rep_hist_padding_count_timers(uint64_t num_timers)
Definition: rephist.c:2800
time_t rep_hist_hs_stats_write(time_t now, bool is_v3)
Definition: rephist.c:2741
time_t rep_hist_buffer_stats_write(time_t now)
Definition: rephist.c:2044
static uint64_t conn_num_created_v4[2][CONN_TYPE_MAX_]
Definition: rephist.c:1696
uint64_t rep_hist_get_circuit_n_handshake_dropped(uint16_t type)
Definition: rephist.c:2430
void rep_hist_make_router_pessimal(const char *id, time_t when)
Definition: rephist.c:757
void rep_hist_exit_stats_term(void)
Definition: rephist.c:1414
void rep_hist_note_overload(overload_type_t overload)
Definition: rephist.c:538
#define STABILITY_INTERVAL
Definition: rephist.c:106
static int32_t overload_onionskin_ntor_period_secs
Definition: rephist.c:2268
STATIC bool should_collect_v3_stats(void)
Definition: rephist.c:2585
uint64_t rep_hist_get_exit_stream_seen(unsigned int cmd)
Definition: rephist.c:1673
void rep_hist_log_circuit_handshake_stats(time_t now)
Definition: rephist.c:2437
void rep_hist_note_circuit_handshake_assigned(uint16_t type)
Definition: rephist.c:2385
#define MIN_CELL_COUNTS_TO_PUBLISH
Definition: rephist.c:2878
static void rep_hist_reset_hs_v3_stats(time_t now)
Definition: rephist.c:2561
void rep_hist_note_tcp_exhaustion(void)
Definition: rephist.c:575
void rep_hist_padding_count_read(padding_type_t type)
Definition: rephist.c:2842
int rep_hist_have_measured_enough_stability(void)
Definition: rephist.c:932
static void rep_hist_reset_desc_stats(time_t now)
Definition: rephist.c:2105
uint64_t rep_hist_get_n_write_limit_reached(void)
Definition: rephist.c:465
static void overload_general_onionskin_assessment(void)
Definition: rephist.c:2322
time_t rep_hist_exit_stats_write(time_t now)
Definition: rephist.c:1590
static double get_stability(or_history_t *hist, time_t when)
Definition: rephist.c:816
Header file for rephist.c.
padding_type_t
Definition: rephist.h:148
@ PADDING_TYPE_DROP
Definition: rephist.h:150
@ PADDING_TYPE_ENABLED_CELL
Definition: rephist.h:158
@ PADDING_TYPE_TOTAL
Definition: rephist.h:154
@ PADDING_TYPE_ENABLED_TOTAL
Definition: rephist.h:156
@ PADDING_TYPE_CELL
Definition: rephist.h:152
overload_type_t
Definition: rephist.h:174
#define REPHIST_CELL_PADDING_COUNTS_INTERVAL
Definition: rephist.h:162
#define MAX_ONION_STAT_TYPE
Definition: rephist.h:106
int tor_sscanf(const char *buf, const char *pattern,...)
Definition: scanf.c:309
void smartlist_add_asprintf(struct smartlist_t *sl, const char *pattern,...)
Definition: smartlist.c:36
char * smartlist_join_strings(smartlist_t *sl, const char *join, int terminate, size_t *len_out)
Definition: smartlist.c:279
void smartlist_sort(smartlist_t *sl, int(*compare)(const void **a, const void **b))
Definition: smartlist.c:334
smartlist_t * smartlist_new(void)
void smartlist_add(smartlist_t *sl, void *element)
void smartlist_clear(smartlist_t *sl)
#define SMARTLIST_FOREACH_BEGIN(sl, type, var)
#define SMARTLIST_FOREACH(sl, type, var, cmd)
int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep, int flags, int max)
double mean_num_cells_in_queue
Definition: rephist.c:1836
uint32_t processed_cells
Definition: rephist.c:1840
double mean_time_cells_in_queue
Definition: rephist.c:1838
struct timeval timestamp_created
Definition: circuit_st.h:169
Definition: node_st.h:34
uint64_t total_cell_waiting_time
Definition: or_circuit_st.h:83
uint32_t processed_cells
Definition: or_circuit_st.h:78
double total_run_weights
Definition: rephist.c:134
unsigned long weighted_run_length
Definition: rephist.c:129
time_t since
Definition: rephist.c:115
uint16_t last_reached_port
Definition: rephist.c:124
time_t start_of_run
Definition: rephist.c:132
time_t changed
Definition: rephist.c:117
tor_addr_t last_reached_addr
Definition: rephist.c:121
uint64_t BandwidthRate
uint64_t BandwidthBurst
uint64_t stats_n_request
Definition: rephist.c:235
time_t next_assessment_time
Definition: rephist.c:240
uint64_t maximum_chanpad_timers
Definition: rephist.c:166
uint64_t read_pad_cell_count
Definition: rephist.c:150
uint64_t enabled_read_pad_cell_count
Definition: rephist.c:158
uint64_t write_pad_cell_count
Definition: rephist.c:152
char first_published_at[ISO_TIME_LEN+1]
Definition: rephist.c:168
uint64_t enabled_write_cell_count
Definition: rephist.c:156
uint64_t write_drop_cell_count
Definition: rephist.c:164
uint64_t write_cell_count
Definition: rephist.c:148
uint64_t read_cell_count
Definition: rephist.c:146
uint64_t enabled_read_cell_count
Definition: rephist.c:154
uint64_t enabled_write_pad_cell_count
Definition: rephist.c:160
uint64_t read_drop_cell_count
Definition: rephist.c:162
#define STATIC
Definition: testsupport.h:32
#define MOCK_IMPL(rv, funcname, arglist)
Definition: testsupport.h:133
int parse_iso_time(const char *cp, time_t *t)
Definition: time_fmt.c:423
void format_iso_time(char *buf, time_t t)
Definition: time_fmt.c:326
void format_local_iso_time(char *buf, time_t t)
Definition: time_fmt.c:316
#define tor_assert(expr)
Definition: util_bug.h:102
int strcmpstart(const char *s1, const char *s2)
Definition: util_string.c:215
int tor_digest_is_zero(const char *digest)
Definition: util_string.c:96