1
//! Implement a cache for onion descriptors and the facility to remember a bit
2
//! about onion service history.
3

            
4
use std::fmt::Debug;
5
use std::mem;
6
use std::panic::AssertUnwindSafe;
7
use std::sync::{Arc, Mutex, MutexGuard};
8
use std::time::{Duration, Instant};
9

            
10
use futures::task::{SpawnError, SpawnExt as _};
11
use futures::FutureExt as _;
12

            
13
use async_trait::async_trait;
14
use educe::Educe;
15
use either::Either::{self, *};
16
use postage::stream::Stream as _;
17
use tracing::{debug, error, trace};
18

            
19
use safelog::sensitive as sv;
20
use tor_basic_utils::define_accessor_trait;
21
use tor_circmgr::isolation::Isolation;
22
use tor_error::{debug_report, error_report, internal, Bug, ErrorReport as _};
23
use tor_hscrypto::pk::HsId;
24
use tor_netdir::NetDir;
25
use tor_rtcompat::Runtime;
26

            
27
use crate::isol_map;
28
use crate::{ConnError, HsClientConnector, HsClientSecretKeys};
29

            
30
slotmap::new_key_type! {
31
    struct TableIndex;
32
}
33

            
34
/// Configuration, currently just some retry parameters
35
#[derive(Default, Debug)]
36
// This is not really public.
37
// It has to be `pub` because it appears in one of the methods in `MockableConnectorData`.
38
// That has to be because that trait is a bound on a parameter for `HsClientConnector`.
39
// `Config` is not re-exported.  (This is isomorphic to the trait sealing pattern.)
40
//
41
// This means that this struct cannot live in the crate root, so we put it here.
42
pub struct Config {
43
    /// Retry parameters
44
    pub(crate) retry: tor_circmgr::CircuitTiming,
45
}
46

            
47
define_accessor_trait! {
48
    /// Configuration for an HS client connector
49
    ///
50
    /// If the HS client connector gains new configurabilities, this trait will gain additional
51
    /// supertraits, as an API break.
52
    ///
53
    /// Prefer to use `TorClientConfig`, which will always implement this trait.
54
    //
55
    // This arrangement is very like that for `CircMgrConfig`.
56
    pub trait HsClientConnectorConfig {
57
        circuit_timing: tor_circmgr::CircuitTiming,
58
    }
59
}
60

            
61
/// Number of times we're willing to iterate round the state machine loop
62
///
63
/// **Not** the number of retries of failed descriptor downloads, circuits, etc.
64
///
65
/// The state machine loop is a condition variable loop.
66
/// It repeatedly transforms the [`ServiceState`] to try to get to `Open`,
67
/// converting stale data to `Closed` and `Closed` to `Working`, and so on.
68
/// This ought only to go forwards so in principle we could use an infinite loop.
69
/// But if we have a logic error, we want to crash eventually.
70
/// The `rechecks` counter is for detecting such a situation.
71
///
72
/// This is fairly arbitrary, but we shouldn't get anywhere near it.
73
///
74
/// Note that this is **not** a number of operational retries
75
/// of fallible retriable operations.
76
/// Such retries are handled in [`connect.rs`](crate::connect).
77
const MAX_RECHECKS: u32 = 10;
78

            
79
/// C Tor `MaxCircuitDirtiness`
80
///
81
/// As per
82
///    <https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914433>
83
///
84
/// And C Tor's `tor(1)`, which says:
85
///
86
/// > MaxCircuitDirtiness NUM
87
/// >
88
/// > Feel free to reuse a circuit that was first used at most NUM
89
/// > seconds ago, but never attach a new stream to a circuit that is
90
/// > too old.  For hidden services, this applies to the last time a
91
/// > circuit was used, not the first.  Circuits with streams
92
/// > constructed with SOCKS authentication via SocksPorts that have
93
/// > KeepAliveIsolateSOCKSAuth also remain alive for
94
/// > MaxCircuitDirtiness seconds after carrying the last such
95
/// > stream. (Default: 10 minutes)
96
///
97
/// However, we're not entirely sure this is the right behaviour.
98
/// See <https://gitlab.torproject.org/tpo/core/arti/-/issues/916>
99
///
100
// TODO SPEC: Explain C Tor `MaxCircuitDirtiness` behaviour
101
//
102
// TODO HS CFG: This should be configurable somehow
103
const RETAIN_CIRCUIT_AFTER_LAST_USE: Duration = Duration::from_secs(10 * 60);
104

            
105
/// How long to retain cached data about a hidden service
106
///
107
/// This is simply to reclaim space, not for correctness.
108
/// So we only check this during housekeeping, not operation.
109
///
110
/// The starting point for this interval is the last time we used the data,
111
/// or a circuit derived from it.
112
///
113
/// Note that this is a *maximum* for the length of time we will retain a descriptor;
114
/// HS descriptors' lifetimes (as declared in the descriptor) *are* honoured;
115
/// but that's done by the code in `connect.rs`, not here.
116
///
117
/// We're not sure this is the right value.
118
/// See <https://gitlab.torproject.org/tpo/core/arti/-/issues/916>
119
//
120
// TODO SPEC: State how long IPT and descriptor data should be retained after use
121
//
122
// TODO HS CFG: Perhaps this should be configurable somehow?
123
const RETAIN_DATA_AFTER_LAST_USE: Duration = Duration::from_secs(48 * 3600 /*hours*/);
124

            
125
/// Hidden services;, our connections to them, and history of connections, etc.
126
///
127
/// Table containing state of our ideas about services.
128
/// Data structure is keyed (indexed) by:
129
///  * `HsId`, hidden service identity
130
///  * any secret keys we are to use
131
///  * circuit isolation
132
///
133
/// We treat different values for any of the above as completely independent,
134
/// except that we try isolation joining (narrowing) if everything else matches.
135
///
136
/// In other words,
137
///  * Two HS connection requests cannot share state and effort
138
///    (descriptor downloads, descriptors, intro pt history)
139
///    unless the client authg keys to be used are the same.
140
///  * This criterion is checked before looking at isolations,
141
///    which may further restrict sharing:
142
///    Two HS connection requests will only share state subject to isolations.
143
///
144
/// Here "state and effort" includes underlying circuits such as hsdir circuits,
145
/// since each HS connection state will use `launch_specific_isolated` for those.
146
#[derive(Default, Debug)]
147
pub(crate) struct Services<D: MockableConnectorData> {
148
    /// The actual records of our connections/attempts for each service, as separated
149
    records: isol_map::MultikeyIsolatedMap<TableIndex, HsId, HsClientSecretKeys, ServiceState<D>>,
150

            
151
    /// Configuration
152
    ///
153
    /// `Arc` so that it can be shared with individual hs connector tasks
154
    config: Arc<Config>,
155
}
156

            
157
/// Entry in the 2nd-level lookup array
158
#[allow(dead_code)] // This alias is here for documentation if nothing else
159
type ServiceRecord<D> = isol_map::Record<HsClientSecretKeys, ServiceState<D>>;
160

            
161
/// Value in the `Services` data structure
162
///
163
/// State and history of of our connections, including connection to any connection task.
164
///
165
/// `last_used` is used to expire data eventually.
166
//
167
// TODO unify this with channels and circuits.  See arti#778.
168
#[derive(Educe)]
169
#[educe(Debug)]
170
enum ServiceState<D: MockableConnectorData> {
171
    /// We don't have a circuit
172
    Closed {
173
        /// The state
174
        data: D,
175
        /// Last time we touched this, including reuse
176
        last_used: Instant,
177
    },
178
    /// We have an open circuit, which we can (hopefully) just use
179
    Open {
180
        /// The state
181
        data: D,
182
        /// The circuit
183
        #[educe(Debug(ignore))]
184
        circuit: Arc<D::ClientCirc>,
185
        /// Last time we touched this, including reuse
186
        ///
187
        /// This is set when we created the circuit, and updated when we
188
        /// hand out this circuit again in response to a new request.
189
        ///
190
        /// We believe this mirrors C Tor behaviour;
191
        /// see [`RETAIN_CIRCUIT_AFTER_LAST_USE`].
192
        last_used: Instant,
193
        /// We have a task that will close the circuit when required
194
        ///
195
        /// This field serves to require construction sites of Open
196
        /// to demonstrate that there *is* an expiry task.
197
        /// In the future, it may also serve to cancel old expiry tasks.
198
        circuit_expiry_task: CircuitExpiryTask,
199
    },
200
    /// We have a task trying to find the service and establish the circuit
201
    ///
202
    /// CachedData is owned by the task.
203
    Working {
204
        /// Signals instances of `get_or_launch_connection` when the task completes
205
        barrier_recv: postage::barrier::Receiver,
206
        /// Where the task will store the error.
207
        ///
208
        /// Lock hierarchy: this lock is "inside" the big lock on `Services`.
209
        error: Arc<Mutex<Option<ConnError>>>,
210
    },
211
    /// Dummy value for use with temporary mem replace
212
    Dummy,
213
}
214

            
215
impl<D: MockableConnectorData> ServiceState<D> {
216
    /// Make a new (blank) `ServiceState::Closed`
217
    fn blank(runtime: &impl Runtime) -> Self {
218
        ServiceState::Closed {
219
            data: D::default(),
220
            last_used: runtime.now(),
221
        }
222
    }
223
}
224

            
225
/// "Continuation" return type from `obtain_circuit_or_continuation_info`
226
type Continuation = (Arc<Mutex<Option<ConnError>>>, postage::barrier::Receiver);
227

            
228
/// Represents a task which is waiting to see when the circuit needs to be expired
229
///
230
/// TODO: Replace this with a task handle that cancels the task when dropped.
231
/// Until then, if the circuit is closed before then, the expiry task will
232
/// uselessly wake up some time later.
233
#[derive(Debug)] // Not Clone
234
struct CircuitExpiryTask {}
235
// impl Drop already, partly to allow explicit drop(CircuitExpiryTask) without clippy complaint
236
impl Drop for CircuitExpiryTask {
237
    fn drop(&mut self) {}
238
}
239

            
240
/// Obtain a circuit from the `Services` table, or return a continuation
241
///
242
/// This is the workhorse function for `get_or_launch_connection`.
243
///
244
/// `get_or_launch_connection`, together with `obtain_circuit_or_continuation_info`,
245
/// form a condition variable loop:
246
///
247
/// We check to see if we have a circuit.  If so, we return it.
248
/// Otherwise, we make sure that a circuit is being constructed,
249
/// and then go into a condvar wait;
250
/// we'll be signaled when the construction completes.
251
///
252
/// So the connection task we spawn does not return the circuit, or error,
253
/// via an inter-task stream.
254
/// It stores it in the data structure and wakes up all the client tasks.
255
/// (This means there is only one success path for the client task code.)
256
///
257
/// There are some wrinkles:
258
///
259
/// ### Existence of this as a separate function
260
///
261
/// The usual structure for a condition variable loop would be something like this:
262
///
263
/// ```rust,ignore
264
/// loop {
265
///    test state and maybe break;
266
///    cv.wait(guard).await; // consumes guard, unlocking after enqueueing us as a waiter
267
///    guard = lock();
268
/// }
269
/// ```
270
///
271
/// However, Rust does not currently understand that the mutex is not
272
/// actually a captured variable held across an await point,
273
/// when the variable is consumed before the await, and re-stored afterwards.
274
/// As a result, the async future becomes erroneously `!Send`:
275
/// <https://github.com/rust-lang/rust/issues/104883>.
276
/// We want the unstable feature `-Zdrop-tracking`:
277
/// <https://github.com/rust-lang/rust/issues/97331>.
278
///
279
/// Instead, to convince the compiler, we must use a scope-based drop of the mutex guard.
280
/// That means converting the "test state and maybe break" part into a sub-function.
281
/// That's what this function is.
282
///
283
/// It returns `Right` if the loop should be exited, returning the circuit to the caller.
284
/// It returns `Left` if the loop needs to do a condition variable wait.
285
///
286
/// ### We're using a barrier as a condition variable
287
///
288
/// We want to be signaled when the task exits.  Indeed, *only* when it exits.
289
/// This functionality is most conveniently in a `postage::barrier`.
290
///
291
/// ### Nested loops
292
///
293
/// Sometimes we want to go round again *without* unlocking.
294
/// Sometimes we must unlock and wait and relock.
295
///
296
/// The drop tracking workaround (see above) means we have to do these two
297
/// in separate scopes.
298
/// So there are two nested loops: one here, and one in `get_or_launch_connection`.
299
/// They both use the same backstop rechecks counter.
300
fn obtain_circuit_or_continuation_info<D: MockableConnectorData>(
301
    connector: &HsClientConnector<impl Runtime, D>,
302
    netdir: &Arc<NetDir>,
303
    hsid: &HsId,
304
    secret_keys: &HsClientSecretKeys,
305
    table_index: TableIndex,
306
    rechecks: &mut impl Iterator,
307
    mut guard: MutexGuard<'_, Services<D>>,
308
) -> Result<Either<Continuation, Arc<D::ClientCirc>>, ConnError> {
309
    let blank_state = || ServiceState::blank(&connector.runtime);
310

            
311
    for _recheck in rechecks {
312
        let record = guard
313
            .records
314
            .by_index_mut(table_index)
315
            .ok_or_else(|| internal!("guard table entry vanished!"))?;
316
        let state = &mut **record;
317

            
318
        trace!("HS conn state: {state:?}");
319

            
320
        let (data, barrier_send) = match state {
321
            ServiceState::Open {
322
                data: _,
323
                circuit,
324
                last_used,
325
                circuit_expiry_task: _,
326
            } => {
327
                let now = connector.runtime.now();
328
                if !D::circuit_is_ok(circuit) {
329
                    // Well that's no good, we need a fresh one, but keep the data
330
                    let data = match mem::replace(state, ServiceState::Dummy) {
331
                        ServiceState::Open {
332
                            data,
333
                            last_used: _,
334
                            circuit: _,
335
                            circuit_expiry_task: _,
336
                        } => data,
337
                        _ => panic!("state changed between matches"),
338
                    };
339
                    *state = ServiceState::Closed {
340
                        data,
341
                        last_used: now,
342
                    };
343
                    continue;
344
                }
345
                *last_used = now;
346
                // No need to tell expiry task about revised expiry time;
347
                // it will see the new last_used when it wakes up at the old expiry time.
348

            
349
                return Ok::<_, ConnError>(Right(circuit.clone()));
350
            }
351
            ServiceState::Working {
352
                barrier_recv,
353
                error,
354
            } => {
355
                if !matches!(
356
                    barrier_recv.try_recv(),
357
                    Err(postage::stream::TryRecvError::Pending)
358
                ) {
359
                    // This information is stale; the task no longer exists.
360
                    // We want information from a fresh task.
361
                    *state = blank_state();
362
                    continue;
363
                }
364
                let barrier_recv = barrier_recv.clone();
365

            
366
                // This clone of the error field Arc<Mutex<..>> allows us to collect errors
367
                // which happened due to the currently-running task, which we have just
368
                // found exists.  Ie, it will see errors that occurred after we entered
369
                // `get_or_launch`.  Stale errors, from previous tasks, were cleared above.
370
                let error = error.clone();
371

            
372
                // Wait for the task to complete (at which point it drops the barrier)
373
                return Ok(Left((error, barrier_recv)));
374
            }
375
            ServiceState::Closed { .. } => {
376
                let (barrier_send, barrier_recv) = postage::barrier::channel();
377
                let data = match mem::replace(
378
                    state,
379
                    ServiceState::Working {
380
                        barrier_recv,
381
                        error: Arc::new(Mutex::new(None)),
382
                    },
383
                ) {
384
                    ServiceState::Closed { data, .. } => data,
385
                    _ => panic!("state changed between matches"),
386
                };
387
                (data, barrier_send)
388
            }
389
            ServiceState::Dummy => {
390
                *state = blank_state();
391
                return Err(internal!("HS connector found dummy state").into());
392
            }
393
        };
394

            
395
        // Make a connection
396
        let runtime = &connector.runtime;
397
        let connector = (*connector).clone();
398
        let config = guard.config.clone();
399
        let netdir = netdir.clone();
400
        let secret_keys = secret_keys.clone();
401
        let hsid = *hsid;
402
        let connect_future = async move {
403
            let mut data = data;
404

            
405
            let got = AssertUnwindSafe(D::connect(
406
                &connector,
407
                netdir,
408
                config,
409
                hsid,
410
                &mut data,
411
                secret_keys,
412
            ))
413
            .catch_unwind()
414
            .await
415
            .unwrap_or_else(|_| {
416
                data = D::default();
417
                Err(internal!("hidden service connector task panicked!").into())
418
            });
419
            let now = connector.runtime.now();
420
            let last_used = now;
421

            
422
            let got = got.and_then(|circuit| {
423
                let circuit_expiry_task = ServiceState::spawn_circuit_expiry_task(
424
                    &connector,
425
                    hsid,
426
                    table_index,
427
                    last_used,
428
                    now,
429
                )
430
                .map_err(|cause| ConnError::Spawn {
431
                    spawning: "circuit expiry task",
432
                    cause: cause.into(),
433
                })?;
434
                Ok((circuit, circuit_expiry_task))
435
            });
436

            
437
            let got_error = got.as_ref().map(|_| ()).map_err(Clone::clone);
438

            
439
            // block for handling inability to store
440
            let stored = async {
441
                let mut guard = connector.services()?;
442
                let record = guard
443
                    .records
444
                    .by_index_mut(table_index)
445
                    .ok_or_else(|| internal!("HS table entry removed while task running"))?;
446
                // Always match this, so we check what we're overwriting
447
                let state = &mut **record;
448
                let error_store = match state {
449
                    ServiceState::Working { error, .. } => error,
450
                    _ => return Err(internal!("HS task found state other than Working")),
451
                };
452

            
453
                match got {
454
                    Ok((circuit, circuit_expiry_task)) => {
455
                        *state = ServiceState::Open {
456
                            data,
457
                            circuit,
458
                            last_used,
459
                            circuit_expiry_task,
460
                        }
461
                    }
462
                    Err(error) => {
463
                        let mut error_store = error_store
464
                            .lock()
465
                            .map_err(|_| internal!("Working error poisoned, cannot store error"))?;
466
                        *error_store = Some(error);
467
                    }
468
                };
469

            
470
                Ok(())
471
            }
472
            .await;
473

            
474
            match (got_error, stored) {
475
                (Ok::<(), ConnError>(()), Ok::<(), Bug>(())) => {}
476
                (Err(got_error), Ok(())) => {
477
                    debug_report!(got_error, "HS connection failure for {}", sv(hsid));
478
                }
479
                (Ok(()), Err(bug)) => {
480
                    error_report!(
481
                        bug,
482
                        "internal error storing built HS circuit for {}",
483
                        sv(hsid)
484
                    );
485
                }
486
                (Err(got_error), Err(bug)) => {
487
                    // We're reporting two errors, so we'll construct the event
488
                    // manually.
489
                    error!(
490
                        "internal error storing HS connection error for {}: {}; {}",
491
                        sv(hsid),
492
                        got_error.report(),
493
                        bug.report(),
494
                    );
495
                }
496
            };
497
            drop(barrier_send);
498
        };
499
        runtime
500
            .spawn_obj(Box::new(connect_future).into())
501
            .map_err(|cause| ConnError::Spawn {
502
                spawning: "connection task",
503
                cause: cause.into(),
504
            })?;
505
    }
506

            
507
    Err(internal!("HS connector state management malfunction (exceeded MAX_RECHECKS").into())
508
}
509

            
510
impl<D: MockableConnectorData> Services<D> {
511
    /// Create a new empty `Services`
512
7
    pub(crate) fn new(config: Config) -> Self {
513
7
        Services {
514
7
            records: Default::default(),
515
7
            config: Arc::new(config),
516
7
        }
517
7
    }
518

            
519
    /// Connect to a hidden service
520
    // We *do* drop guard.  There is *one* await point, just after drop(guard).
521
    pub(crate) async fn get_or_launch_connection(
522
        connector: &HsClientConnector<impl Runtime, D>,
523
        netdir: &Arc<NetDir>,
524
        hs_id: HsId,
525
        isolation: Box<dyn Isolation>,
526
        secret_keys: HsClientSecretKeys,
527
    ) -> Result<Arc<D::ClientCirc>, ConnError> {
528
        let blank_state = || ServiceState::blank(&connector.runtime);
529

            
530
        let mut rechecks = 0..MAX_RECHECKS;
531

            
532
        let mut obtain = |table_index, guard| {
533
            obtain_circuit_or_continuation_info(
534
                connector,
535
                netdir,
536
                &hs_id,
537
                &secret_keys,
538
                table_index,
539
                &mut rechecks,
540
                guard,
541
            )
542
        };
543

            
544
        let mut got;
545
        let table_index;
546
        {
547
            let mut guard = connector.services()?;
548
            let services = &mut *guard;
549

            
550
            trace!("HS conn get_or_launch: {hs_id:?} {isolation:?} {secret_keys:?}");
551
            //trace!("HS conn services: {services:?}");
552

            
553
            table_index =
554
                services
555
                    .records
556
                    .index_or_insert_with(&hs_id, &secret_keys, isolation, blank_state);
557

            
558
            let guard = guard;
559
            got = obtain(table_index, guard);
560
        }
561
        loop {
562
            // The parts of this loop which run after a `Left` is returned
563
            // logically belong in the case in `obtain_circuit_or_continuation_info`
564
            // for `ServiceState::Working`, where that function decides we need to wait.
565
            // This code has to be out here to help the compiler's drop tracking.
566
            {
567
                // Block to scope the acquisition of `error`, a guard
568
                // for the mutex-protected error field in the state,
569
                // and, for neatness, barrier_recv.
570

            
571
                let (error, mut barrier_recv) = match got? {
572
                    Right(ret) => return Ok(ret),
573
                    Left(continuation) => continuation,
574
                };
575

            
576
                barrier_recv.recv().await;
577

            
578
                let error = error
579
                    .lock()
580
                    .map_err(|_| internal!("Working error poisoned"))?;
581
                if let Some(error) = &*error {
582
                    return Err(error.clone());
583
                }
584
            }
585

            
586
            let guard = connector.services()?;
587

            
588
            got = obtain(table_index, guard);
589
        }
590
    }
591

            
592
    /// Perform housekeeping - delete data we aren't interested in any more
593
7
    pub(crate) fn run_housekeeping(&mut self, now: Instant) {
594
7
        self.expire_old_data(now);
595
7
    }
596

            
597
    /// Delete data we aren't interested in any more
598
7
    fn expire_old_data(&mut self, now: Instant) {
599
7
        self.records
600
7
            .retain(|hsid, record, _table_index| match &**record {
601
                ServiceState::Closed { data: _, last_used } => {
602
                    let Some(expiry_time) = last_used.checked_add(RETAIN_DATA_AFTER_LAST_USE)
603
                    else {
604
                        return false;
605
                    };
606
                    now <= expiry_time
607
                }
608
                ServiceState::Open { .. } | ServiceState::Working { .. } => true,
609
                ServiceState::Dummy { .. } => {
610
                    error!("found dummy data during HS housekeeping, for {}", sv(hsid));
611
                    false
612
                }
613
7
            });
614
7
    }
615
}
616

            
617
impl<D: MockableConnectorData> ServiceState<D> {
618
    /// Spawn a task that will drop our reference to the rendezvous circuit
619
    /// at `table_index` when it has gone too long without any use.
620
    ///
621
    /// According to [`RETAIN_CIRCUIT_AFTER_LAST_USE`].
622
    //
623
    // As it happens, this function is always called with `last_used` equal to `now`,
624
    // but we pass separate arguments for clarity.
625
    fn spawn_circuit_expiry_task(
626
        connector: &HsClientConnector<impl Runtime, D>,
627
        hsid: HsId,
628
        table_index: TableIndex,
629
        last_used: Instant,
630
        now: Instant,
631
    ) -> Result<CircuitExpiryTask, SpawnError> {
632
        /// Returns the duration until expiry, or `None` if it should expire now
633
        fn calculate_expiry_wait(last_used: Instant, now: Instant) -> Option<Duration> {
634
            let expiry = last_used
635
                .checked_add(RETAIN_CIRCUIT_AFTER_LAST_USE)
636
                .or_else(|| {
637
                    error!("time overflow calculating HS circuit expiry, killing circuit!");
638
                    None
639
                })?;
640
            let wait = expiry.checked_duration_since(now).unwrap_or_default();
641
            if wait == Duration::ZERO {
642
                return None;
643
            }
644
            Some(wait)
645
        }
646

            
647
        let mut maybe_wait = calculate_expiry_wait(last_used, now);
648
        let () = connector.runtime.spawn({
649
            let connector = connector.clone();
650
            async move {
651
                // This loop is slightly odd.  The wait ought naturally to be at the end,
652
                // but that would mean a useless re-lock and re-check right after creation,
653
                // or jumping into the middle of the loop.
654
                loop {
655
                    if let Some(yes_wait) = maybe_wait {
656
                        connector.runtime.sleep(yes_wait).await;
657
                    }
658
                    // If it's None, we can't rely on that to say we should expire it,
659
                    // since that information crossed a time when we didn't hold the lock.
660

            
661
                    let Ok(mut guard) = connector.services() else {
662
                        break;
663
                    };
664
                    let Some(record) = guard.records.by_index_mut(table_index) else {
665
                        break;
666
                    };
667
                    let state = &mut **record;
668
                    let last_used = match state {
669
                        ServiceState::Closed { .. } => break,
670
                        ServiceState::Open { last_used, .. } => *last_used,
671
                        ServiceState::Working { .. } => break, // someone else will respawn
672
                        ServiceState::Dummy => break,          // someone else will (report and) fix
673
                    };
674
                    maybe_wait = calculate_expiry_wait(last_used, connector.runtime.now());
675
                    if maybe_wait.is_none() {
676
                        match mem::replace(state, ServiceState::Dummy) {
677
                            ServiceState::Open {
678
                                data,
679
                                circuit,
680
                                last_used,
681
                                circuit_expiry_task,
682
                            } => {
683
                                debug!("HS connection expires: {hsid}");
684
                                drop(circuit);
685
                                drop(circuit_expiry_task); // that's us
686
                                *state = ServiceState::Closed { data, last_used };
687
                                break;
688
                            }
689
                            _ => panic!("state now {state:?} even though we just saw it Open"),
690
                        }
691
                    }
692
                }
693
            }
694
        })?;
695
        Ok(CircuitExpiryTask {})
696
    }
697
}
698

            
699
/// Mocking for actual HS connection work, to let us test the `Services` state machine
700
//
701
// Does *not* mock circmgr, chanmgr, etc. - those won't be used by the tests, since our
702
// `connect` won't call them.  But mocking them pollutes many types with `R` and is
703
// generally tiresome.  So let's not.  Instead the tests can make dummy ones.
704
//
705
// This trait is actually crate-private, since it isn't re-exported, but it must
706
// be `pub` because it appears as a default for a type parameter in HsClientConnector.
707
#[async_trait]
708
pub trait MockableConnectorData: Default + Debug + Send + Sync + 'static {
709
    /// Client circuit
710
    type ClientCirc: Sync + Send + 'static;
711

            
712
    /// Mock state
713
    type MockGlobalState: Clone + Sync + Send + 'static;
714

            
715
    /// Connect
716
    async fn connect<R: Runtime>(
717
        connector: &HsClientConnector<R, Self>,
718
        netdir: Arc<NetDir>,
719
        config: Arc<Config>,
720
        hsid: HsId,
721
        data: &mut Self,
722
        secret_keys: HsClientSecretKeys,
723
    ) -> Result<Arc<Self::ClientCirc>, ConnError>;
724

            
725
    /// Is circuit OK?  Ie, not `.is_closing()`.
726
    fn circuit_is_ok(circuit: &Self::ClientCirc) -> bool;
727
}
728

            
729
#[cfg(test)]
730
pub(crate) mod test {
731
    // @@ begin test lint list maintained by maint/add_warning @@
732
    #![allow(clippy::bool_assert_comparison)]
733
    #![allow(clippy::clone_on_copy)]
734
    #![allow(clippy::dbg_macro)]
735
    #![allow(clippy::print_stderr)]
736
    #![allow(clippy::print_stdout)]
737
    #![allow(clippy::single_char_pattern)]
738
    #![allow(clippy::unwrap_used)]
739
    #![allow(clippy::unchecked_duration_subtraction)]
740
    #![allow(clippy::useless_vec)]
741
    #![allow(clippy::needless_pass_by_value)]
742
    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
743
    use super::*;
744
    use crate::*;
745
    use futures::{poll, SinkExt};
746
    use std::fmt;
747
    use std::task::Poll::{self, *};
748
    use tokio::pin;
749
    use tokio_crate as tokio;
750
    use tor_rtcompat::{test_with_one_runtime, SleepProvider};
751
    use tor_rtmock::MockRuntime;
752
    use tracing_test::traced_test;
753

            
754
    use ConnError as E;
755

            
756
    #[derive(Debug, Default)]
757
    struct MockData {
758
        connect_called: usize,
759
    }
760

            
761
    /// Type indicating what our `connect()` should return; it always makes a fresh MockCirc
762
    type MockGive = Poll<Result<(), E>>;
763

            
764
    #[derive(Debug, Clone)]
765
    struct MockGlobalState {
766
        // things will appear here when we have more sophisticated tests
767
        give: postage::watch::Receiver<MockGive>,
768
    }
769

            
770
    #[derive(Clone, Educe)]
771
    #[educe(Debug)]
772
    struct MockCirc {
773
        #[educe(Debug(method = "debug_arc_mutex"))]
774
        ok: Arc<Mutex<bool>>,
775
        connect_called: usize,
776
    }
777

            
778
    fn debug_arc_mutex(val: &Arc<Mutex<impl Debug>>, f: &mut fmt::Formatter) -> fmt::Result {
779
        write!(f, "@{:?}", Arc::as_ptr(val))?;
780
        let guard = val.lock();
781
        let guard = guard.or_else(|g| {
782
            write!(f, ",POISON")?;
783
            Ok::<_, fmt::Error>(g.into_inner())
784
        })?;
785
        write!(f, " ")?;
786
        Debug::fmt(&*guard, f)
787
    }
788

            
789
    impl PartialEq for MockCirc {
790
        fn eq(&self, other: &MockCirc) -> bool {
791
            Arc::ptr_eq(&self.ok, &other.ok)
792
        }
793
    }
794

            
795
    impl MockCirc {
796
        fn new(connect_called: usize) -> Self {
797
            let ok = Arc::new(Mutex::new(true));
798
            MockCirc { ok, connect_called }
799
        }
800
    }
801

            
802
    #[async_trait]
803
    impl MockableConnectorData for MockData {
804
        type ClientCirc = MockCirc;
805
        type MockGlobalState = MockGlobalState;
806

            
807
        async fn connect<R: Runtime>(
808
            connector: &HsClientConnector<R, MockData>,
809
            _netdir: Arc<NetDir>,
810
            _config: Arc<Config>,
811
            _hsid: HsId,
812
            data: &mut MockData,
813
            _secret_keys: HsClientSecretKeys,
814
        ) -> Result<Arc<Self::ClientCirc>, E> {
815
            data.connect_called += 1;
816
            let make = {
817
                let connect_called = data.connect_called;
818
                move |()| Arc::new(MockCirc::new(connect_called))
819
            };
820
            let mut give = connector.mock_for_state.give.clone();
821
            if let Ready(ret) = &*give.borrow() {
822
                return ret.clone().map(make);
823
            }
824
            loop {
825
                match give.recv().await.expect("EOF on mock_global_state stream") {
826
                    Pending => {}
827
                    Ready(ret) => return ret.map(make),
828
                }
829
            }
830
        }
831

            
832
        fn circuit_is_ok(circuit: &Self::ClientCirc) -> bool {
833
            *circuit.ok.lock().unwrap()
834
        }
835
    }
836

            
837
    /// Makes a non-empty `HsClientSecretKeys`, containing (somehow) `kk`
838
    fn mk_keys(kk: u8) -> HsClientSecretKeys {
839
        let mut ss = [0_u8; 32];
840
        ss[0] = kk;
841
        let keypair = tor_llcrypto::pk::ed25519::Keypair::from_bytes(&ss);
842
        let mut b = HsClientSecretKeysBuilder::default();
843
        #[allow(deprecated)]
844
        b.ks_hsc_intro_auth(keypair.into());
845
        b.build().unwrap()
846
    }
847

            
848
    fn mk_hsconn<R: Runtime>(
849
        runtime: R,
850
    ) -> (
851
        HsClientConnector<R, MockData>,
852
        HsClientSecretKeys,
853
        postage::watch::Sender<MockGive>,
854
    ) {
855
        let chanmgr = tor_chanmgr::ChanMgr::new(
856
            runtime.clone(),
857
            &Default::default(),
858
            tor_chanmgr::Dormancy::Dormant,
859
            &Default::default(),
860
        );
861
        let guardmgr = tor_guardmgr::GuardMgr::new(
862
            runtime.clone(),
863
            tor_persist::TestingStateMgr::new(),
864
            &tor_guardmgr::TestConfig::default(),
865
        )
866
        .unwrap();
867
        let circmgr = tor_circmgr::CircMgr::new(
868
            &tor_circmgr::TestConfig::default(),
869
            tor_persist::TestingStateMgr::new(),
870
            &runtime,
871
            Arc::new(chanmgr),
872
            guardmgr,
873
        )
874
        .unwrap();
875
        let circpool = HsCircPool::new(&circmgr);
876
        let (give_send, give) = postage::watch::channel_with(Ready(Ok(())));
877
        let mock_for_state = MockGlobalState { give };
878
        #[allow(clippy::let_and_return)] // we'll probably add more in this function
879
        let hscc = HsClientConnector {
880
            runtime,
881
            circpool,
882
            services: Default::default(),
883
            mock_for_state,
884
        };
885
        let keys = HsClientSecretKeysBuilder::default().build().unwrap();
886
        (hscc, keys, give_send)
887
    }
888

            
889
    #[allow(clippy::unnecessary_wraps)]
890
    fn mk_isol(s: &str) -> Option<NarrowableIsolation> {
891
        Some(NarrowableIsolation(s.into()))
892
    }
893

            
894
    async fn launch_one(
895
        hsconn: &HsClientConnector<impl Runtime, MockData>,
896
        id: u8,
897
        secret_keys: &HsClientSecretKeys,
898
        isolation: Option<NarrowableIsolation>,
899
    ) -> Result<Arc<MockCirc>, ConnError> {
900
        let netdir = tor_netdir::testnet::construct_netdir()
901
            .unwrap_if_sufficient()
902
            .unwrap();
903
        let netdir = Arc::new(netdir);
904

            
905
        let hs_id = {
906
            let mut hs_id = [0_u8; 32];
907
            hs_id[0] = id;
908
            hs_id.into()
909
        };
910
        #[allow(clippy::redundant_closure)] // srsly, that would be worse
911
        let isolation = isolation.unwrap_or_default().into();
912
        Services::get_or_launch_connection(hsconn, &netdir, hs_id, isolation, secret_keys.clone())
913
            .await
914
    }
915

            
916
    #[derive(Default, Debug, Clone)]
917
    // TODO move this to tor-circmgr under a test feature?
918
    pub(crate) struct NarrowableIsolation(pub(crate) String);
919
    impl tor_circmgr::isolation::IsolationHelper for NarrowableIsolation {
920
        fn compatible_same_type(&self, other: &Self) -> bool {
921
            self.join_same_type(other).is_some()
922
        }
923
        fn join_same_type(&self, other: &Self) -> Option<Self> {
924
            Some(if self.0.starts_with(&other.0) {
925
                self.clone()
926
            } else if other.0.starts_with(&self.0) {
927
                other.clone()
928
            } else {
929
                return None;
930
            })
931
        }
932
    }
933

            
934
    #[test]
935
    #[traced_test]
936
    fn simple() {
937
        test_with_one_runtime!(|runtime| async {
938
            let (hsconn, keys, _give_send) = mk_hsconn(runtime);
939

            
940
            let circuit = launch_one(&hsconn, 0, &keys, None).await.unwrap();
941
            eprintln!("{:?}", circuit);
942
        });
943
    }
944

            
945
    #[test]
946
    #[traced_test]
947
    fn expiry() {
948
        MockRuntime::test_with_various(|runtime| async move {
949
            // This is the amount by which we adjust clock advances to make sure we
950
            // hit more or less than a particular value, to avoid edge cases and
951
            // cope with real time advancing too.
952
            // This does *not* represent an actual delay to real test runs.
953
            const TIMEOUT_SLOP: Duration = Duration::from_secs(10);
954

            
955
            let (hsconn, keys, _give_send) = mk_hsconn(runtime.clone());
956

            
957
            let advance = |duration| {
958
                let hsconn = hsconn.clone();
959
                let runtime = &runtime;
960
                async move {
961
                    // let expiry task get going and choose its expiry (wakeup) time
962
                    runtime.progress_until_stalled().await;
963
                    // TODO: Make this use runtime.advance_by() when that's not very slow
964
                    runtime.mock_sleep().advance(duration);
965
                    // let expiry task run
966
                    runtime.progress_until_stalled().await;
967
                    hsconn.services().unwrap().run_housekeeping(runtime.now());
968
                }
969
            };
970

            
971
            // make circuit1
972
            let circuit1 = launch_one(&hsconn, 0, &keys, None).await.unwrap();
973

            
974
            // expire it
975
            advance(RETAIN_CIRCUIT_AFTER_LAST_USE + TIMEOUT_SLOP).await;
976

            
977
            // make circuit2 (a)
978
            let circuit2a = launch_one(&hsconn, 0, &keys, None).await.unwrap();
979
            assert_ne!(circuit1, circuit2a);
980

            
981
            // nearly expire it, then reuse it
982
            advance(RETAIN_CIRCUIT_AFTER_LAST_USE - TIMEOUT_SLOP).await;
983
            let circuit2b = launch_one(&hsconn, 0, &keys, None).await.unwrap();
984
            assert_eq!(circuit2a, circuit2b);
985

            
986
            // nearly expire it again, then reuse it
987
            advance(RETAIN_CIRCUIT_AFTER_LAST_USE - TIMEOUT_SLOP).await;
988
            let circuit2c = launch_one(&hsconn, 0, &keys, None).await.unwrap();
989
            assert_eq!(circuit2a, circuit2c);
990

            
991
            // actually expire it
992
            advance(RETAIN_CIRCUIT_AFTER_LAST_USE + TIMEOUT_SLOP).await;
993
            let circuit3 = launch_one(&hsconn, 0, &keys, None).await.unwrap();
994
            assert_ne!(circuit2c, circuit3);
995
            assert_eq!(circuit3.connect_called, 3);
996

            
997
            advance(RETAIN_DATA_AFTER_LAST_USE + Duration::from_secs(10)).await;
998
            let circuit4 = launch_one(&hsconn, 0, &keys, None).await.unwrap();
999
            assert_eq!(circuit4.connect_called, 1);
        });
    }
    #[test]
    #[traced_test]
    fn coalesce() {
        test_with_one_runtime!(|runtime| async {
            let (hsconn, keys, mut give_send) = mk_hsconn(runtime);
            give_send.send(Pending).await.unwrap();
            let c1f = launch_one(&hsconn, 0, &keys, None);
            pin!(c1f);
            for _ in 0..10 {
                assert!(poll!(&mut c1f).is_pending());
            }
            // c2f will find Working
            let c2f = launch_one(&hsconn, 0, &keys, None);
            pin!(c2f);
            for _ in 0..10 {
                assert!(poll!(&mut c1f).is_pending());
                assert!(poll!(&mut c2f).is_pending());
            }
            give_send.send(Ready(Ok(()))).await.unwrap();
            let c1 = c1f.await.unwrap();
            let c2 = c2f.await.unwrap();
            assert_eq!(c1, c2);
            // c2 will find Open
            let c3 = launch_one(&hsconn, 0, &keys, None).await.unwrap();
            assert_eq!(c1, c3);
            assert_ne!(c1, launch_one(&hsconn, 1, &keys, None).await.unwrap());
            assert_ne!(
                c1,
                launch_one(&hsconn, 0, &mk_keys(42), None).await.unwrap()
            );
            let c_isol_1 = launch_one(&hsconn, 0, &keys, mk_isol("a")).await.unwrap();
            assert_eq!(c1, c_isol_1); // We can reuse, but now we've narrowed the isol
            let c_isol_2 = launch_one(&hsconn, 0, &keys, mk_isol("b")).await.unwrap();
            assert_ne!(c1, c_isol_2);
        });
    }
}