1
//! Main implementation of the connection functionality
2

            
3
use std::time::Duration;
4

            
5
use std::collections::HashMap;
6
use std::fmt::Debug;
7
use std::marker::PhantomData;
8
use std::sync::Arc;
9
use std::time::Instant;
10

            
11
use async_trait::async_trait;
12
use educe::Educe;
13
use futures::{AsyncRead, AsyncWrite};
14
use itertools::Itertools;
15
use rand::Rng;
16
use tor_bytes::Writeable;
17
use tor_cell::relaycell::hs::intro_payload::{self, IntroduceHandshakePayload};
18
use tor_cell::relaycell::hs::pow::ProofOfWork;
19
use tor_cell::relaycell::msg::{AnyRelayMsg, Introduce1, Rendezvous2};
20
use tor_circmgr::build::onion_circparams_from_netparams;
21
use tor_error::{debug_report, warn_report, Bug};
22
use tor_hscrypto::Subcredential;
23
use tor_proto::circuit::handshake::hs_ntor;
24
use tracing::{debug, trace};
25

            
26
use retry_error::RetryError;
27
use safelog::Sensitive;
28
use tor_cell::relaycell::hs::{
29
    AuthKeyType, EstablishRendezvous, IntroduceAck, RendezvousEstablished,
30
};
31
use tor_cell::relaycell::RelayMsg;
32
use tor_checkable::{timed::TimerangeBound, Timebound};
33
use tor_circmgr::hspool::{HsCircKind, HsCircPool};
34
use tor_circmgr::timeouts::Action as TimeoutsAction;
35
use tor_dirclient::request::Requestable as _;
36
use tor_error::{internal, into_internal};
37
use tor_error::{HasRetryTime as _, RetryTime};
38
use tor_hscrypto::pk::{HsBlindId, HsId, HsIdKey};
39
use tor_hscrypto::RendCookie;
40
use tor_linkspec::{CircTarget, HasRelayIds, OwnedCircTarget, RelayId};
41
use tor_llcrypto::pk::ed25519::Ed25519Identity;
42
use tor_netdir::{NetDir, Relay};
43
use tor_netdoc::doc::hsdesc::{HsDesc, IntroPointDesc};
44
use tor_proto::circuit::{CircParameters, ClientCirc, MetaCellDisposition, MsgHandler};
45
use tor_rtcompat::{Runtime, SleepProviderExt as _, TimeoutError};
46

            
47
use crate::pow::HsPowClient;
48
use crate::proto_oneshot;
49
use crate::relay_info::ipt_to_circtarget;
50
use crate::state::MockableConnectorData;
51
use crate::Config;
52
use crate::{rend_pt_identity_for_error, FailedAttemptError, IntroPtIndex, RendPtIdentityForError};
53
use crate::{ConnError, DescriptorError, DescriptorErrorDetail};
54
use crate::{HsClientConnector, HsClientSecretKeys};
55

            
56
use ConnError as CE;
57
use FailedAttemptError as FAE;
58

            
59
/// Number of hops in our hsdir, introduction, and rendezvous circuits
60
///
61
/// Required by `tor_circmgr`'s timeout estimation API
62
/// ([`tor_circmgr::CircMgr::estimate_timeout`], [`HsCircPool::estimate_timeout`]).
63
///
64
/// TODO HS hardcoding the number of hops to 3 seems wrong.
65
/// This is really something that HsCircPool knows.  And some setups might want to make
66
/// shorter circuits for some reason.  And it will become wrong with vanguards?
67
/// But right now I think this is what HsCircPool does.
68
//
69
// Some commentary from
70
//   https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/1342#note_2918050
71
// Possibilities:
72
//  * Look at n_hops() on the circuits we get, if we don't need this estimate
73
//    till after we have the circuit.
74
//  * Add a function to HsCircPool to tell us what length of circuit to expect
75
//    for each given type of circuit.
76
const HOPS: usize = 3;
77

            
78
/// Given `R, M` where `M: MocksForConnect<M>`, expand to the mockable `ClientCirc`
79
// This is quite annoying.  But the alternative is to write out `<... as // ...>`
80
// each time, since otherwise the compile complains about ambiguous associated types.
81
macro_rules! ClientCirc { { $R:ty, $M:ty } => {
82
    <<$M as MocksForConnect<$R>>::HsCircPool as MockableCircPool<$R>>::ClientCirc
83
} }
84

            
85
/// Information about a hidden service, including our connection history
86
#[derive(Default, Educe)]
87
#[educe(Debug)]
88
// This type is actually crate-private, since it isn't re-exported, but it must
89
// be `pub` because it appears as a default for a type parameter in HsClientConnector.
90
pub struct Data {
91
    /// The latest known onion service descriptor for this service.
92
    desc: DataHsDesc,
93
    /// Information about the latest status of trying to connect to this service
94
    /// through each of its introduction points.
95
    ipts: DataIpts,
96
}
97

            
98
/// Part of `Data` that relates to the HS descriptor
99
type DataHsDesc = Option<TimerangeBound<HsDesc>>;
100

            
101
/// Part of `Data` that relates to our information about introduction points
102
type DataIpts = HashMap<RelayIdForExperience, IptExperience>;
103

            
104
/// How things went last time we tried to use this introduction point
105
///
106
/// Neither this data structure, nor [`Data`], is responsible for arranging that we expire this
107
/// information eventually.  If we keep reconnecting to the service, we'll retain information
108
/// about each IPT indefinitely, at least so long as they remain listed in the descriptors we
109
/// receive.
110
///
111
/// Expiry of unused data is handled by `state.rs`, according to `last_used` in `ServiceState`.
112
///
113
/// Choosing which IPT to prefer is done by obtaining an `IptSortKey`
114
/// (from this and other information).
115
//
116
// Don't impl Ord for IptExperience.  We obtain `Option<&IptExperience>` from our
117
// data structure, and if IptExperience were Ord then Option<&IptExperience> would be Ord
118
// but it would be the wrong sort order: it would always prefer None, ie untried IPTs.
119
#[derive(Debug)]
120
struct IptExperience {
121
    /// How long it took us to get whatever outcome occurred
122
    ///
123
    /// We prefer fast successes to slow ones.
124
    /// Then, we prefer failures with earlier `RetryTime`,
125
    /// and, lastly, faster failures to slower ones.
126
    duration: Duration,
127

            
128
    /// What happened and when we might try again
129
    ///
130
    /// Note that we don't actually *enforce* the `RetryTime` here, just sort by it
131
    /// using `RetryTime::loose_cmp`.
132
    ///
133
    /// We *do* return an error that is itself `HasRetryTime` and expect our callers
134
    /// to honour that.
135
    outcome: Result<(), RetryTime>,
136
}
137

            
138
/// Actually make a HS connection, updating our recorded state as necessary
139
///
140
/// `connector` is provided only for obtaining the runtime and netdir (and `mock_for_state`).
141
/// Obviously, `connect` is not supposed to go looking in `services`.
142
///
143
/// This function handles all necessary retrying of fallible operations,
144
/// (and, therefore, must also limit the total work done for a particular call).
145
///
146
/// This function has a minimum of functionality, since it is the boundary
147
/// between "mock connection, used for testing `state.rs`" and
148
/// "mock circuit and netdir, used for testing `connect.rs`",
149
/// so it is not, itself, unit-testable.
150
pub(crate) async fn connect<R: Runtime>(
151
    connector: &HsClientConnector<R>,
152
    netdir: Arc<NetDir>,
153
    config: Arc<Config>,
154
    hsid: HsId,
155
    data: &mut Data,
156
    secret_keys: HsClientSecretKeys,
157
) -> Result<Arc<ClientCirc>, ConnError> {
158
    Context::new(
159
        &connector.runtime,
160
        &*connector.circpool,
161
        netdir,
162
        config,
163
        hsid,
164
        secret_keys,
165
        (),
166
    )?
167
    .connect(data)
168
    .await
169
}
170

            
171
/// Common context for a single request to connect to a hidden service
172
///
173
/// This saves on passing this same set of (immutable) values (or subsets thereof)
174
/// to each method in the principal functional code, everywhere.
175
/// It also provides a convenient type to be `Self`.
176
///
177
/// Its lifetime is one request to make a new client circuit to a hidden service,
178
/// including all the retries and timeouts.
179
struct Context<'c, R: Runtime, M: MocksForConnect<R>> {
180
    /// Runtime
181
    runtime: &'c R,
182
    /// Circpool
183
    circpool: &'c M::HsCircPool,
184
    /// Netdir
185
    //
186
    // TODO holding onto the netdir for the duration of our attempts is not ideal
187
    // but doing better is fairly complicated.  See discussions here:
188
    //   https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/1228#note_2910545
189
    //   https://gitlab.torproject.org/tpo/core/arti/-/issues/884
190
    netdir: Arc<NetDir>,
191
    /// Configuration
192
    config: Arc<Config>,
193
    /// Secret keys to use
194
    secret_keys: HsClientSecretKeys,
195
    /// HS ID
196
    hsid: HsId,
197
    /// Blinded HS ID
198
    hs_blind_id: HsBlindId,
199
    /// The subcredential to use during this time period
200
    subcredential: Subcredential,
201
    /// Mock data
202
    mocks: M,
203
}
204

            
205
/// Details of an established rendezvous point
206
///
207
/// Intermediate value for progress during a connection attempt.
208
struct Rendezvous<'r, R: Runtime, M: MocksForConnect<R>> {
209
    /// RPT as a `Relay`
210
    rend_relay: Relay<'r>,
211
    /// Rendezvous circuit
212
    rend_circ: Arc<ClientCirc!(R, M)>,
213
    /// Rendezvous cookie
214
    rend_cookie: RendCookie,
215

            
216
    /// Receiver that will give us the RENDEZVOUS2 message.
217
    ///
218
    /// The sending ended is owned by the handler
219
    /// which receives control messages on the rendezvous circuit,
220
    /// and which was installed when we sent `ESTABLISH_RENDEZVOUS`.
221
    ///
222
    /// (`RENDEZVOUS2` is the message containing the onion service's side of the handshake.)
223
    rend2_rx: proto_oneshot::Receiver<Rendezvous2>,
224

            
225
    /// Dummy, to placate compiler
226
    ///
227
    /// Covariant without dropck or interfering with Send/Sync will do fine.
228
    marker: PhantomData<fn() -> (R, M)>,
229
}
230

            
231
/// Random value used as part of IPT selection
232
type IptSortRand = u32;
233

            
234
/// Details of an apparently-useable introduction point
235
///
236
/// Intermediate value for progress during a connection attempt.
237
struct UsableIntroPt<'i> {
238
    /// Index in HS descriptor
239
    intro_index: IntroPtIndex,
240
    /// IPT descriptor
241
    intro_desc: &'i IntroPointDesc,
242
    /// IPT `CircTarget`
243
    intro_target: OwnedCircTarget,
244
    /// Random value used as part of IPT selection
245
    sort_rand: IptSortRand,
246
}
247

            
248
/// Lookup key for looking up and recording our IPT use experiences
249
///
250
/// Used to identify a relay when looking to see what happened last time we used it,
251
/// and storing that information after we tried it.
252
///
253
/// We store the experience information under an arbitrary one of the relay's identities,
254
/// as returned by the `HasRelayIds::identities().next()`.
255
/// When we do lookups, we check all the relay's identities to see if we find
256
/// anything relevant.
257
/// If relay identities permute in strange ways, whether we find our previous
258
/// knowledge about them is not particularly well defined, but that's fine.
259
///
260
/// While this is, structurally, a relay identity, it is not suitable for other purposes.
261
#[derive(Hash, Eq, PartialEq, Ord, PartialOrd, Debug)]
262
struct RelayIdForExperience(RelayId);
263

            
264
/// Details of an apparently-successful INTRODUCE exchange
265
///
266
/// Intermediate value for progress during a connection attempt.
267
struct Introduced<R: Runtime, M: MocksForConnect<R>> {
268
    /// End-to-end crypto NTORv3 handshake with the service
269
    ///
270
    /// Created as part of generating our `INTRODUCE1`,
271
    /// and then used when processing `RENDEZVOUS2`.
272
    handshake_state: hs_ntor::HsNtorClientState,
273

            
274
    /// Dummy, to placate compiler
275
    ///
276
    /// `R` and `M` only used for getting to mocks.
277
    /// Covariant without dropck or interfering with Send/Sync will do fine.
278
    marker: PhantomData<fn() -> (R, M)>,
279
}
280

            
281
impl RelayIdForExperience {
282
    /// Identities to use to try to find previous experience information about this IPT
283
12
    fn for_lookup(intro_target: &OwnedCircTarget) -> impl Iterator<Item = Self> + '_ {
284
12
        intro_target
285
12
            .identities()
286
30
            .map(|id| RelayIdForExperience(id.to_owned()))
287
12
    }
288

            
289
    /// Identity to use to store previous experience information about this IPT
290
    fn for_store(intro_target: &OwnedCircTarget) -> Result<Self, Bug> {
291
        let id = intro_target
292
            .identities()
293
            .next()
294
            .ok_or_else(|| internal!("introduction point relay with no identities"))?
295
            .to_owned();
296
        Ok(RelayIdForExperience(id))
297
    }
298
}
299

            
300
/// Sort key for an introduction point, for selecting the best IPTs to try first
301
///
302
/// Ordering is most preferable first.
303
///
304
/// We use this to sort our `UsableIpt`s using `.sort_by_key`.
305
/// (This implementation approach ensures that we obey all the usual ordering invariants.)
306
#[derive(Ord, PartialOrd, Eq, PartialEq, Debug)]
307
struct IptSortKey {
308
    /// Sort by how preferable the experience was
309
    outcome: IptSortKeyOutcome,
310
    /// Failing that, choose randomly
311
    sort_rand: IptSortRand,
312
}
313

            
314
/// Component of the [`IptSortKey`] representing outcome of our last attempt, if any
315
///
316
/// This is the main thing we use to decide which IPTs to try first.
317
/// It is calculated for each IPT
318
/// (via `.sort_by_key`, so repeatedly - it should therefore be cheap to make.)
319
///
320
/// Ordering is most preferable first.
321
#[derive(Ord, PartialOrd, Eq, PartialEq, Debug)]
322
enum IptSortKeyOutcome {
323
    /// Prefer successes
324
    Success {
325
        /// Prefer quick ones
326
        duration: Duration,
327
    },
328
    /// Failing that, try one we don't know to have failed
329
    Untried,
330
    /// Failing that, it'll have to be ones that didn't work last time
331
    Failed {
332
        /// Prefer failures with an earlier retry time
333
        retry_time: tor_error::LooseCmpRetryTime,
334
        /// Failing that, prefer quick failures (rather than slow ones eg timeouts)
335
        duration: Duration,
336
    },
337
}
338

            
339
impl From<Option<&IptExperience>> for IptSortKeyOutcome {
340
12
    fn from(experience: Option<&IptExperience>) -> IptSortKeyOutcome {
341
        use IptSortKeyOutcome as O;
342
12
        match experience {
343
12
            None => O::Untried,
344
            Some(IptExperience { duration, outcome }) => match outcome {
345
                Ok(()) => O::Success {
346
                    duration: *duration,
347
                },
348
                Err(retry_time) => O::Failed {
349
                    retry_time: (*retry_time).into(),
350
                    duration: *duration,
351
                },
352
            },
353
        }
354
12
    }
355
}
356

            
357
impl<'c, R: Runtime, M: MocksForConnect<R>> Context<'c, R, M> {
358
    /// Make a new `Context` from the input data
359
2
    fn new(
360
2
        runtime: &'c R,
361
2
        circpool: &'c M::HsCircPool,
362
2
        netdir: Arc<NetDir>,
363
2
        config: Arc<Config>,
364
2
        hsid: HsId,
365
2
        secret_keys: HsClientSecretKeys,
366
2
        mocks: M,
367
2
    ) -> Result<Self, ConnError> {
368
2
        let time_period = netdir.hs_time_period();
369
2
        let (hs_blind_id_key, subcredential) = HsIdKey::try_from(hsid)
370
2
            .map_err(|_| CE::InvalidHsId)?
371
2
            .compute_blinded_key(time_period)
372
2
            .map_err(
373
2
                // TODO HS what on earth do these errors mean, in practical terms ?
374
2
                // In particular, we'll want to convert them to a ConnError variant,
375
2
                // but what ErrorKind should they have ?
376
2
                into_internal!("key blinding error, don't know how to handle"),
377
2
            )?;
378
2
        let hs_blind_id = hs_blind_id_key.id();
379
2

            
380
2
        Ok(Context {
381
2
            netdir,
382
2
            config,
383
2
            hsid,
384
2
            hs_blind_id,
385
2
            subcredential,
386
2
            circpool,
387
2
            runtime,
388
2
            secret_keys,
389
2
            mocks,
390
2
        })
391
2
    }
392

            
393
    /// Actually make a HS connection, updating our recorded state as necessary
394
    ///
395
    /// Called by the `connect` function in this module.
396
    ///
397
    /// This function handles all necessary retrying of fallible operations,
398
    /// (and, therefore, must also limit the total work done for a particular call).
399
2
    async fn connect(&self, data: &mut Data) -> Result<Arc<ClientCirc!(R, M)>, ConnError> {
400
2
        // This function must do the following, retrying as appropriate.
401
2
        //  - Look up the onion descriptor in the state.
402
2
        //  - Download the onion descriptor if one isn't there.
403
2
        //  - In parallel:
404
2
        //    - Pick a rendezvous point from the netdirprovider and launch a
405
2
        //      rendezvous circuit to it. Then send ESTABLISH_INTRO.
406
2
        //    - Pick a number of introduction points (1 or more) and try to
407
2
        //      launch circuits to them.
408
2
        //  - On a circuit to an introduction point, send an INTRODUCE1 cell.
409
2
        //  - Wait for a RENDEZVOUS2 cell on the rendezvous circuit
410
2
        //  - Add a virtual hop to the rendezvous circuit.
411
2
        //  - Return the rendezvous circuit.
412
2

            
413
2
        let mocks = self.mocks.clone();
414

            
415
2
        let desc = self.descriptor_ensure(&mut data.desc).await?;
416

            
417
2
        mocks.test_got_desc(desc);
418

            
419
2
        let circ = self.intro_rend_connect(desc, &mut data.ipts).await?;
420
        mocks.test_got_circ(&circ);
421

            
422
        Ok(circ)
423
    }
424

            
425
    /// Ensure that `Data.desc` contains the HS descriptor
426
    ///
427
    /// If we have a previously-downloaded descriptor, which is still valid,
428
    /// just returns a reference to it.
429
    ///
430
    /// Otherwise, tries to obtain the descriptor by downloading it from hsdir(s).
431
    ///
432
    /// Does all necessary retries and timeouts.
433
    /// Returns an error if no valid descriptor could be found.
434
2
    async fn descriptor_ensure<'d>(&self, data: &'d mut DataHsDesc) -> Result<&'d HsDesc, CE> {
435
2
        // Maximum number of hsdir connection and retrieval attempts we'll make
436
2
        let max_total_attempts = self
437
2
            .config
438
2
            .retry
439
2
            .hs_desc_fetch_attempts()
440
2
            .try_into()
441
2
            // User specified a very large u32.  We must be downcasting it to 16bit!
442
2
            // let's give them as many retries as we can manage.
443
2
            .unwrap_or(usize::MAX);
444
2

            
445
2
        // Limit on the duration of each retrieval attempt
446
2
        let each_timeout = self.estimate_timeout(&[
447
2
            (1, TimeoutsAction::BuildCircuit { length: HOPS }), // build circuit
448
2
            (1, TimeoutsAction::RoundTrip { length: HOPS }),    // One HTTP query/response
449
2
        ]);
450

            
451
        // We retain a previously obtained descriptor precisely until its lifetime expires,
452
        // and pay no attention to the descriptor's revision counter.
453
        // When it expires, we discard it completely and try to obtain a new one.
454
        //   https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914448
455
        // TODO SPEC: Discuss HS descriptor lifetime and expiry client behaviour
456
2
        if let Some(previously) = data {
457
            let now = self.runtime.wallclock();
458
            if let Ok(_desc) = previously.as_ref().check_valid_at(&now) {
459
                // Ideally we would just return desc but that confuses borrowck.
460
                // https://github.com/rust-lang/rust/issues/51545
461
                return Ok(data
462
                    .as_ref()
463
                    .expect("Some but now None")
464
                    .as_ref()
465
                    .check_valid_at(&now)
466
                    .expect("Ok but now Err"));
467
            }
468
            // Seems to be not valid now.  Try to fetch a fresh one.
469
2
        }
470

            
471
2
        let hs_dirs = self.netdir.hs_dirs_download(
472
2
            self.hs_blind_id,
473
2
            self.netdir.hs_time_period(),
474
2
            &mut self.mocks.thread_rng(),
475
2
        )?;
476

            
477
2
        trace!(
478
            "HS desc fetch for {}, using {} hsdirs",
479
            &self.hsid,
480
            hs_dirs.len()
481
        );
482

            
483
        // We might consider launching requests to multiple HsDirs in parallel.
484
        //   https://gitlab.torproject.org/tpo/core/arti/-/merge_requests/1118#note_2894463
485
        // But C Tor doesn't and our HS experts don't consider that important:
486
        //   https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914436
487
        // (Additionally, making multiple HSDir requests at once may make us
488
        // more vulnerable to traffic analysis.)
489
2
        let mut attempts = hs_dirs.iter().cycle().take(max_total_attempts);
490
2
        let mut errors = RetryError::in_attempt_to("retrieve hidden service descriptor");
491
2
        let desc = loop {
492
2
            let relay = match attempts.next() {
493
2
                Some(relay) => relay,
494
                None => {
495
                    return Err(if errors.is_empty() {
496
                        CE::NoHsDirs
497
                    } else {
498
                        CE::DescriptorDownload(errors)
499
                    })
500
                }
501
            };
502
2
            let hsdir_for_error: Sensitive<Ed25519Identity> = (*relay.id()).into();
503
2
            match self
504
2
                .runtime
505
2
                .timeout(each_timeout, self.descriptor_fetch_attempt(relay))
506
2
                .await
507
2
                .unwrap_or(Err(DescriptorErrorDetail::Timeout))
508
            {
509
2
                Ok(desc) => break desc,
510
                Err(error) => {
511
                    debug_report!(
512
                        &error,
513
                        "failed hsdir desc fetch for {} from {}/{}",
514
                        &self.hsid,
515
                        &relay.id(),
516
                        &relay.rsa_id()
517
                    );
518
                    errors.push(tor_error::Report(DescriptorError {
519
                        hsdir: hsdir_for_error,
520
                        error,
521
                    }));
522
                }
523
            }
524
        };
525

            
526
        // Store the bounded value in the cache for reuse,
527
        // but return a reference to the unwrapped `HsDesc`.
528
        //
529
        // The `HsDesc` must be owned by `data.desc`,
530
        // so first add it to `data.desc`,
531
        // and then dangerously_assume_timely to get a reference out again.
532
        //
533
        // It is safe to dangerously_assume_timely,
534
        // as descriptor_fetch_attempt has already checked the timeliness of the descriptor.
535
2
        let ret = data.insert(desc);
536
2
        Ok(ret.as_ref().dangerously_assume_timely())
537
2
    }
538

            
539
    /// Make one attempt to fetch the descriptor from a specific hsdir
540
    ///
541
    /// No timeout
542
    ///
543
    /// On success, returns the descriptor.
544
    ///
545
    /// While the returned descriptor is `TimerangeBound`, its validity at the current time *has*
546
    /// been checked.
547
2
    async fn descriptor_fetch_attempt(
548
2
        &self,
549
2
        hsdir: &Relay<'_>,
550
2
    ) -> Result<TimerangeBound<HsDesc>, DescriptorErrorDetail> {
551
2
        let max_len: usize = self
552
2
            .netdir
553
2
            .params()
554
2
            .hsdir_max_desc_size
555
2
            .get()
556
2
            .try_into()
557
2
            .map_err(into_internal!("BoundedInt was not truly bounded!"))?;
558
2
        let request = {
559
2
            let mut r = tor_dirclient::request::HsDescDownloadRequest::new(self.hs_blind_id);
560
2
            r.set_max_len(max_len);
561
2
            r
562
2
        };
563
2
        trace!(
564
            "hsdir for {}, trying {}/{}, request {:?} (http request {:?})",
565
            &self.hsid,
566
            &hsdir.id(),
567
            &hsdir.rsa_id(),
568
            &request,
569
            request.debug_request()
570
        );
571

            
572
2
        let circuit = self
573
2
            .circpool
574
2
            .m_get_or_launch_specific(
575
2
                &self.netdir,
576
2
                HsCircKind::ClientHsDir,
577
2
                OwnedCircTarget::from_circ_target(hsdir),
578
2
            )
579
2
            .await?;
580
2
        let mut stream = circuit
581
2
            .m_begin_dir_stream()
582
2
            .await
583
2
            .map_err(DescriptorErrorDetail::Stream)?;
584

            
585
2
        let response = tor_dirclient::send_request(self.runtime, &request, &mut stream, None)
586
2
            .await
587
2
            .map_err(|dir_error| match dir_error {
588
                tor_dirclient::Error::RequestFailed(rfe) => DescriptorErrorDetail::from(rfe.error),
589
                tor_dirclient::Error::CircMgr(ce) => into_internal!(
590
                    "tor-dirclient complains about circmgr going wrong but we gave it a stream"
591
                )(ce)
592
                .into(),
593
                other => into_internal!(
594
                    "tor-dirclient gave unexpected error, tor-hsclient code needs updating"
595
                )(other)
596
                .into(),
597
2
            })?;
598

            
599
2
        let desc_text = response.into_output_string().map_err(|rfe| rfe.error)?;
600
2
        let hsc_desc_enc = self.secret_keys.keys.ks_hsc_desc_enc.as_ref();
601
2

            
602
2
        let now = self.runtime.wallclock();
603
2

            
604
2
        HsDesc::parse_decrypt_validate(
605
2
            &desc_text,
606
2
            &self.hs_blind_id,
607
2
            now,
608
2
            &self.subcredential,
609
2
            hsc_desc_enc,
610
2
        )
611
2
        .map_err(DescriptorErrorDetail::from)
612
2
    }
613

            
614
    /// Given the descriptor, try to connect to service
615
    ///
616
    /// Does all necessary retries, timeouts, etc.
617
2
    async fn intro_rend_connect(
618
2
        &self,
619
2
        desc: &HsDesc,
620
2
        data: &mut DataIpts,
621
2
    ) -> Result<Arc<ClientCirc!(R, M)>, CE> {
622
2
        // Maximum number of rendezvous/introduction attempts we'll make
623
2
        let max_total_attempts = self
624
2
            .config
625
2
            .retry
626
2
            .hs_intro_rend_attempts()
627
2
            .try_into()
628
2
            // User specified a very large u32.  We must be downcasting it to 16bit!
629
2
            // let's give them as many retries as we can manage.
630
2
            .unwrap_or(usize::MAX);
631
2

            
632
2
        // Limit on the duration of each attempt to establish a rendezvous point
633
2
        //
634
2
        // This *might* include establishing a fresh circuit,
635
2
        // if the HsCircPool's pool is empty.
636
2
        let rend_timeout = self.estimate_timeout(&[
637
2
            (1, TimeoutsAction::BuildCircuit { length: HOPS }), // build circuit
638
2
            (1, TimeoutsAction::RoundTrip { length: HOPS }),    // One ESTABLISH_RENDEZVOUS
639
2
        ]);
640
2

            
641
2
        // Limit on the duration of each attempt to negotiate with an introduction point
642
2
        //
643
2
        // *Does* include establishing the circuit.
644
2
        let intro_timeout = self.estimate_timeout(&[
645
2
            (1, TimeoutsAction::BuildCircuit { length: HOPS }), // build circuit
646
2
            // This does some crypto too, but we don't account for that.
647
2
            (1, TimeoutsAction::RoundTrip { length: HOPS }), // One INTRODUCE1/INTRODUCE_ACK
648
2
        ]);
649

            
650
        // Timeout estimator for the action that the HS will take in building
651
        // its circuit to the RPT.
652
2
        let hs_build_action = TimeoutsAction::BuildCircuit {
653
2
            length: if desc.is_single_onion_service() {
654
                1
655
            } else {
656
2
                HOPS
657
            },
658
        };
659
        // Limit on the duration of each attempt for activities involving both
660
        // RPT and IPT.
661
2
        let rpt_ipt_timeout = self.estimate_timeout(&[
662
2
            // The API requires us to specify a number of circuit builds and round trips.
663
2
            // So what we tell the estimator is a rather imprecise description.
664
2
            // (TODO it would be nice if the circmgr offered us a one-way trip Action).
665
2
            //
666
2
            // What we are timing here is:
667
2
            //
668
2
            //    INTRODUCE2 goes from IPT to HS
669
2
            //    but that happens in parallel with us waiting for INTRODUCE_ACK,
670
2
            //    which is controlled by `intro_timeout` so not pat of `ipt_rpt_timeout`.
671
2
            //    and which has to come HOPS hops.  So don't count INTRODUCE2 here.
672
2
            //
673
2
            //    HS builds to our RPT
674
2
            (1, hs_build_action),
675
2
            //
676
2
            //    RENDEZVOUS1 goes from HS to RPT.  `hs_hops`, one-way.
677
2
            //    RENDEZVOUS2 goes from RPT to us.  HOPS, one-way.
678
2
            //    Together, we squint a bit and call this a HOPS round trip:
679
2
            (1, TimeoutsAction::RoundTrip { length: HOPS }),
680
2
        ]);
681
2

            
682
2
        // We can't reliably distinguish IPT failure from RPT failure, so we iterate over IPTs
683
2
        // (best first) and each time use a random RPT.
684
2

            
685
2
        // We limit the number of rendezvous establishment attempts, separately, since we don't
686
2
        // try to talk to the intro pt until we've established the rendezvous circuit.
687
2
        let mut rend_attempts = 0..max_total_attempts;
688
2

            
689
2
        // But, we put all the errors into the same bucket, since we might have a mixture.
690
2
        let mut errors = RetryError::in_attempt_to("make circuit to to hidden service");
691
2

            
692
2
        // Note that IntroPtIndex is *not* the index into this Vec.
693
2
        // It is the index into the original list of introduction points in the descriptor.
694
2
        let mut usable_intros: Vec<UsableIntroPt> = desc
695
2
            .intro_points()
696
2
            .iter()
697
2
            .enumerate()
698
6
            .map(|(intro_index, intro_desc)| {
699
6
                let intro_index = intro_index.into();
700
6
                let intro_target = ipt_to_circtarget(intro_desc, &self.netdir)
701
6
                    .map_err(|error| FAE::UnusableIntro { error, intro_index })?;
702
                // Lack of TAIT means this clone
703
6
                let intro_target = OwnedCircTarget::from_circ_target(&intro_target);
704
6
                Ok::<_, FailedAttemptError>(UsableIntroPt {
705
6
                    intro_index,
706
6
                    intro_desc,
707
6
                    intro_target,
708
6
                    sort_rand: self.mocks.thread_rng().random(),
709
6
                })
710
6
            })
711
6
            .filter_map(|entry| match entry {
712
6
                Ok(y) => Some(y),
713
                Err(e) => {
714
                    errors.push(e);
715
                    None
716
                }
717
6
            })
718
2
            .collect_vec();
719
2

            
720
2
        // Delete experience information for now-unlisted intro points
721
2
        // Otherwise, as the IPTs change `Data` might grow without bound,
722
2
        // if we keep reconnecting to the same HS.
723
2
        data.retain(|k, _v| {
724
            usable_intros
725
                .iter()
726
                .any(|ipt| RelayIdForExperience::for_lookup(&ipt.intro_target).any(|id| &id == k))
727
2
        });
728
2

            
729
2
        // Join with existing state recording our experiences,
730
2
        // sort by descending goodness, and then randomly
731
2
        // (so clients without any experience don't all pile onto the same, first, IPT)
732
12
        usable_intros.sort_by_key(|ipt: &UsableIntroPt| {
733
12
            let experience =
734
24
                RelayIdForExperience::for_lookup(&ipt.intro_target).find_map(|id| data.get(&id));
735
12
            IptSortKey {
736
12
                outcome: experience.into(),
737
12
                sort_rand: ipt.sort_rand,
738
12
            }
739
12
        });
740
2
        self.mocks.test_got_ipts(&usable_intros);
741
2

            
742
2
        let mut intro_attempts = usable_intros.iter().cycle().take(max_total_attempts);
743
2

            
744
2
        // We retain a rendezvous we managed to set up in here.  That way if we created it, and
745
2
        // then failed before we actually needed it, we can reuse it.
746
2
        // If we exit with an error, we will waste it - but because we isolate things we do
747
2
        // for different services, it wouldn't be reusable anyway.
748
2
        let mut saved_rendezvous = None;
749
2

            
750
2
        // If we are using proof-of-work DoS mitigation, this chooses an
751
2
        // algorithm and initial effort, and adjusts that effort when we retry.
752
2
        let mut pow_client = HsPowClient::new(&self.hs_blind_id, desc);
753

            
754
        // We might consider making multiple INTRODUCE attempts to different
755
        // IPTs in in parallel, and somehow aggregating the errors and
756
        // experiences.
757
        // However our HS experts don't consider that important:
758
        //   https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914438
759
        // Parallelizing our HsCircPool circuit building would likely have
760
        // greater impact. (See #1149.)
761
        loop {
762
            // When did we start doing things that depended on the IPT?
763
            //
764
            // Used for recording our experience with the selected IPT
765
2
            let mut ipt_use_started = None::<Instant>;
766

            
767
            // Error handling inner async block (analogous to an IEFE):
768
            //  * Ok(Some()) means this attempt succeeded
769
            //  * Ok(None) means all attempts exhausted
770
            //  * Err(error) means this attempt failed
771
            //
772
            // Error handling is rather complex here.  It's the primary job of *this* code to
773
            // make sure that it's done right for timeouts.  (The individual component
774
            // functions handle non-timeout errors.)  The different timeout errors have
775
            // different amounts of information about the identity of the RPT and IPT: in each
776
            // case, the error only mentions the RPT or IPT if that node is implicated in the
777
            // timeout.
778
2
            let outcome = async {
779
2
                // We establish a rendezvous point first.  Although it appears from reading
780
2
                // this code that this means we serialise establishment of the rendezvous and
781
2
                // introduction circuits, this isn't actually the case.  The circmgr maintains
782
2
                // a pool of circuits.  What actually happens in the "standing start" case is
783
2
                // that we obtain a circuit for rendezvous from the circmgr's pool, expecting
784
2
                // one to be available immediately; the circmgr will then start to build a new
785
2
                // one to replenish its pool, and that happens in parallel with the work we do
786
2
                // here - but in arrears.  If the circmgr pool is empty, then we must wait.
787
2
                //
788
2
                // Perhaps this should be parallelised here.  But that's really what the pool
789
2
                // is for, since we expect building the rendezvous circuit and building the
790
2
                // introduction circuit to take about the same length of time.
791
2
                //
792
2
                // We *do* serialise the ESTABLISH_RENDEZVOUS exchange, with the
793
2
                // building of the introduction circuit.  That could be improved, at the cost
794
2
                // of some additional complexity here.
795
2
                //
796
2
                // Our HS experts don't consider it important to increase the parallelism:
797
2
                //   https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914444
798
2
                //   https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914445
799
2
                if saved_rendezvous.is_none() {
800
2
                    debug!("hs conn to {}: setting up rendezvous point", &self.hsid);
801
                    // Establish a rendezvous circuit.
802
2
                    let Some(_): Option<usize> = rend_attempts.next() else {
803
                        return Ok(None);
804
                    };
805

            
806
2
                    let mut using_rend_pt = None;
807
                    saved_rendezvous = Some(
808
2
                        self.runtime
809
2
                            .timeout(rend_timeout, self.establish_rendezvous(&mut using_rend_pt))
810
2
                            .await
811
                            .map_err(|_: TimeoutError| match using_rend_pt {
812
                                None => FAE::RendezvousCircuitObtain {
813
                                    error: tor_circmgr::Error::CircTimeout(None),
814
                                },
815
                                Some(rend_pt) => FAE::RendezvousEstablishTimeout { rend_pt },
816
                            })??,
817
                    );
818
                }
819

            
820
                let Some(ipt) = intro_attempts.next() else {
821
                    return Ok(None);
822
                };
823
                let intro_index = ipt.intro_index;
824

            
825
                let proof_of_work = match pow_client.solve().await {
826
                    Ok(solution) => solution,
827
                    Err(e) => {
828
                        debug!(
829
                            "failing to compute proof-of-work, trying without. ({:?})",
830
                            e
831
                        );
832
                        None
833
                    }
834
                };
835

            
836
                // We record how long things take, starting from here, as
837
                // as a statistic we'll use for the IPT in future.
838
                // This is stored in a variable outside this async block,
839
                // so that the outcome handling can use it.
840
                ipt_use_started = Some(self.runtime.now());
841

            
842
                // No `Option::get_or_try_insert_with`, or we'd avoid this expect()
843
                let rend_pt_for_error = rend_pt_identity_for_error(
844
                    &saved_rendezvous
845
                        .as_ref()
846
                        .expect("just made Some")
847
                        .rend_relay,
848
                );
849
                debug!(
850
                    "hs conn to {}: RPT {}",
851
                    &self.hsid,
852
                    rend_pt_for_error.as_inner()
853
                );
854

            
855
                let (rendezvous, introduced) = self
856
                    .runtime
857
                    .timeout(
858
                        intro_timeout,
859
                        self.exchange_introduce(ipt, &mut saved_rendezvous,
860
                            proof_of_work),
861
                    )
862
                    .await
863
                    .map_err(|_: TimeoutError| {
864
                        // The intro point ought to give us a prompt ACK regardless of HS
865
                        // behaviour or whatever is happening at the RPT, so blame the IPT.
866
                        FAE::IntroductionTimeout { intro_index }
867
                    })?
868
                    // TODO: Maybe try, once, to extend-and-reuse the intro circuit.
869
                    //
870
                    // If the introduction fails, the introduction circuit is in principle
871
                    // still usable.  We believe that in this case, C Tor extends the intro
872
                    // circuit by one hop to the next IPT to try.  That saves on building a
873
                    // whole new 3-hop intro circuit.  However, our HS experts tell us that
874
                    // if introduction fails at one IPT it is likely to fail at the others too,
875
                    // so that optimisation might reduce our network impact and time to failure,
876
                    // but isn't likely to improve our chances of success.
877
                    //
878
                    // However, it's not clear whether this approach risks contaminating
879
                    // the 2nd attempt with some fault relating to the introduction point.
880
                    // The 1st ipt might also gain more knowledge about which HS we're talking to.
881
                    //
882
                    // TODO SPEC: Discuss extend-and-reuse HS intro circuit after nack
883
                    ?;
884
                #[allow(unused_variables)] // it's *supposed* to be unused
885
                let saved_rendezvous = (); // don't use `saved_rendezvous` any more, use rendezvous
886

            
887
                let rend_pt = rend_pt_identity_for_error(&rendezvous.rend_relay);
888
                let circ = self
889
                    .runtime
890
                    .timeout(
891
                        rpt_ipt_timeout,
892
                        self.complete_rendezvous(ipt, rendezvous, introduced),
893
                    )
894
                    .await
895
                    .map_err(|_: TimeoutError| FAE::RendezvousCompletionTimeout {
896
                        intro_index,
897
                        rend_pt: rend_pt.clone(),
898
                    })??;
899

            
900
                debug!(
901
                    "hs conn to {}: RPT {} IPT {}: success",
902
                    &self.hsid,
903
                    rend_pt.as_inner(),
904
                    intro_index,
905
                );
906
                Ok::<_, FAE>(Some((intro_index, circ)))
907
2
            }
908
2
            .await;
909

            
910
            // Store the experience `outcome` we had with IPT `intro_index`, in `data`
911
            #[allow(clippy::unused_unit)] // -> () is here for error handling clarity
912
            let mut store_experience = |intro_index, outcome| -> () {
913
                (|| {
914
                    let ipt = usable_intros
915
                        .iter()
916
                        .find(|ipt| ipt.intro_index == intro_index)
917
                        .ok_or_else(|| internal!("IPT not found by index"))?;
918
                    let id = RelayIdForExperience::for_store(&ipt.intro_target)?;
919
                    let started = ipt_use_started.ok_or_else(|| {
920
                        internal!("trying to record IPT use but no IPT start time noted")
921
                    })?;
922
                    let duration = self
923
                        .runtime
924
                        .now()
925
                        .checked_duration_since(started)
926
                        .ok_or_else(|| internal!("clock overflow calculating IPT use duration"))?;
927
                    data.insert(id, IptExperience { duration, outcome });
928
                    Ok::<_, Bug>(())
929
                })()
930
                .unwrap_or_else(|e| warn_report!(e, "error recording HS IPT use experience"));
931
            };
932

            
933
            match outcome {
934
                Ok(Some((intro_index, y))) => {
935
                    // Record successful outcome in Data
936
                    store_experience(intro_index, Ok(()));
937
                    return Ok(y);
938
                }
939
                Ok(None) => return Err(CE::Failed(errors)),
940
                Err(error) => {
941
                    debug_report!(&error, "hs conn to {}: attempt failed", &self.hsid);
942
                    // Record error outcome in Data, if in fact we involved the IPT
943
                    // at all.  The IPT information is be retrieved from `error`,
944
                    // since only some of the errors implicate the introduction point.
945
                    if let Some(intro_index) = error.intro_index() {
946
                        store_experience(intro_index, Err(error.retry_time()));
947
                    }
948
                    errors.push(error);
949

            
950
                    // If we are using proof-of-work DoS mitigation, try harder next time
951
                    pow_client.increase_effort();
952
                }
953
            }
954
        }
955
    }
956

            
957
    /// Make one attempt to establish a rendezvous circuit
958
    ///
959
    /// This doesn't really depend on anything,
960
    /// other than (obviously) the isolation implied by our circuit pool.
961
    /// In particular it doesn't depend on the introduction point.
962
    ///
963
    /// Does not apply a timeout.
964
    ///
965
    /// On entry `using_rend_pt` is `None`.
966
    /// This function will store `Some` when it finds out which relay
967
    /// it is talking to and starts to converse with it.
968
    /// That way, if a timeout occurs, the caller can add that information to the error.
969
2
    async fn establish_rendezvous(
970
2
        &'c self,
971
2
        using_rend_pt: &mut Option<RendPtIdentityForError>,
972
2
    ) -> Result<Rendezvous<'c, R, M>, FAE> {
973
2
        let (rend_circ, rend_relay) = self
974
2
            .circpool
975
2
            .m_get_or_launch_client_rend(&self.netdir)
976
2
            .await
977
            .map_err(|error| FAE::RendezvousCircuitObtain { error })?;
978

            
979
        let rend_pt = rend_pt_identity_for_error(&rend_relay);
980
        *using_rend_pt = Some(rend_pt.clone());
981

            
982
        let rend_cookie: RendCookie = self.mocks.thread_rng().random();
983
        let message = EstablishRendezvous::new(rend_cookie);
984

            
985
        let (rend_established_tx, rend_established_rx) = proto_oneshot::channel();
986
        let (rend2_tx, rend2_rx) = proto_oneshot::channel();
987

            
988
        /// Handler which expects `RENDEZVOUS_ESTABLISHED` and then
989
        /// `RENDEZVOUS2`.   Returns each message via the corresponding `oneshot`.
990
        struct Handler {
991
            /// Sender for a RENDEZVOUS_ESTABLISHED message.
992
            rend_established_tx: proto_oneshot::Sender<RendezvousEstablished>,
993
            /// Sender for a RENDEZVOUS2 message.
994
            rend2_tx: proto_oneshot::Sender<Rendezvous2>,
995
        }
996
        impl MsgHandler for Handler {
997
            fn handle_msg(
998
                &mut self,
999
                msg: AnyRelayMsg,
            ) -> Result<MetaCellDisposition, tor_proto::Error> {
                // The first message we expect is a RENDEZVOUS_ESTABALISHED.
                if self.rend_established_tx.still_expected() {
                    self.rend_established_tx
                        .deliver_expected_message(msg, MetaCellDisposition::Consumed)
                } else {
                    self.rend2_tx
                        .deliver_expected_message(msg, MetaCellDisposition::ConversationFinished)
                }
            }
        }
        debug!(
            "hs conn to {}: RPT {}: sending ESTABLISH_RENDEZVOUS",
            &self.hsid,
            rend_pt.as_inner(),
        );
        let handle_proto_error = |error| FAE::RendezvousEstablish {
            error,
            rend_pt: rend_pt.clone(),
        };
        let handler = Handler {
            rend_established_tx,
            rend2_tx,
        };
        rend_circ
            .m_start_conversation_last_hop(Some(message.into()), handler)
            .await
            .map_err(handle_proto_error)?;
        // `start_conversation` returns as soon as the control message has been sent.
        // We need to obtain the RENDEZVOUS_ESTABLISHED message, which is "returned" via the oneshot.
        let _: RendezvousEstablished = rend_established_rx.recv(handle_proto_error).await?;
        debug!(
            "hs conn to {}: RPT {}: got RENDEZVOUS_ESTABLISHED",
            &self.hsid,
            rend_pt.as_inner(),
        );
        Ok(Rendezvous {
            rend_circ,
            rend_cookie,
            rend_relay,
            rend2_rx,
            marker: PhantomData,
        })
    }
    /// Attempt (once) to send an INTRODUCE1 and wait for the INTRODUCE_ACK
    ///
    /// `take`s the input `rendezvous` (but only takes it if it gets that far)
    /// and, if successful, returns it.
    /// (This arranges that the rendezvous is "used up" precisely if
    /// we sent its secret somewhere.)
    ///
    /// Although this function handles the `Rendezvous`,
    /// nothing in it actually involves the rendezvous point.
    /// So if there's a failure, it's purely to do with the introduction point.
    ///
    /// Does not apply a timeout.
    async fn exchange_introduce(
        &'c self,
        ipt: &UsableIntroPt<'_>,
        rendezvous: &mut Option<Rendezvous<'c, R, M>>,
        proof_of_work: Option<ProofOfWork>,
    ) -> Result<(Rendezvous<'c, R, M>, Introduced<R, M>), FAE> {
        let intro_index = ipt.intro_index;
        debug!(
            "hs conn to {}: IPT {}: obtaining intro circuit",
            &self.hsid, intro_index,
        );
        let intro_circ = self
            .circpool
            .m_get_or_launch_specific(
                &self.netdir,
                HsCircKind::ClientIntro,
                ipt.intro_target.clone(), // &OwnedCircTarget isn't CircTarget apparently
            )
            .await
            .map_err(|error| FAE::IntroductionCircuitObtain { error, intro_index })?;
        let rendezvous = rendezvous.take().ok_or_else(|| internal!("no rend"))?;
        let rend_pt = rend_pt_identity_for_error(&rendezvous.rend_relay);
        debug!(
            "hs conn to {}: RPT {} IPT {}: making introduction",
            &self.hsid,
            rend_pt.as_inner(),
            intro_index,
        );
        // Now we construct an introduce1 message and perform the first part of the
        // rendezvous handshake.
        //
        // This process is tricky because the header of the INTRODUCE1 message
        // -- which depends on the IntroPt configuration -- is authenticated as
        // part of the HsDesc handshake.
        // Construct the header, since we need it as input to our encryption.
        let intro_header = {
            let ipt_sid_key = ipt.intro_desc.ipt_sid_key();
            let intro1 = Introduce1::new(
                AuthKeyType::ED25519_SHA3_256,
                ipt_sid_key.as_bytes().to_vec(),
                vec![],
            );
            let mut header = vec![];
            intro1
                .encode_onto(&mut header)
                .map_err(into_internal!("couldn't encode intro1 header"))?;
            header
        };
        // Construct the introduce payload, which tells the onion service how to find
        // our rendezvous point.  (We could do this earlier if we wanted.)
        let intro_payload = {
            let onion_key =
                intro_payload::OnionKey::NtorOnionKey(*rendezvous.rend_relay.ntor_onion_key());
            let linkspecs = rendezvous
                .rend_relay
                .linkspecs()
                .map_err(into_internal!("Couldn't encode link specifiers"))?;
            let payload = IntroduceHandshakePayload::new(
                rendezvous.rend_cookie,
                onion_key,
                linkspecs,
                proof_of_work,
            );
            let mut encoded = vec![];
            payload
                .write_onto(&mut encoded)
                .map_err(into_internal!("Couldn't encode introduce1 payload"))?;
            encoded
        };
        // Perform the cryptographic handshake with the onion service.
        let service_info = hs_ntor::HsNtorServiceInfo::new(
            ipt.intro_desc.svc_ntor_key().clone(),
            ipt.intro_desc.ipt_sid_key().clone(),
            self.subcredential,
        );
        let handshake_state =
            hs_ntor::HsNtorClientState::new(&mut self.mocks.thread_rng(), service_info);
        let encrypted_body = handshake_state
            .client_send_intro(&intro_header, &intro_payload)
            .map_err(into_internal!("can't begin hs-ntor handshake"))?;
        // Build our actual INTRODUCE1 message.
        let intro1_real = Introduce1::new(
            AuthKeyType::ED25519_SHA3_256,
            ipt.intro_desc.ipt_sid_key().as_bytes().to_vec(),
            encrypted_body,
        );
        /// Handler which expects just `INTRODUCE_ACK`
        struct Handler {
            /// Sender for `INTRODUCE_ACK`
            intro_ack_tx: proto_oneshot::Sender<IntroduceAck>,
        }
        impl MsgHandler for Handler {
            fn handle_msg(
                &mut self,
                msg: AnyRelayMsg,
            ) -> Result<MetaCellDisposition, tor_proto::Error> {
                self.intro_ack_tx
                    .deliver_expected_message(msg, MetaCellDisposition::ConversationFinished)
            }
        }
        let handle_intro_proto_error = |error| FAE::IntroductionExchange { error, intro_index };
        let (intro_ack_tx, intro_ack_rx) = proto_oneshot::channel();
        let handler = Handler { intro_ack_tx };
        debug!(
            "hs conn to {}: RPT {} IPT {}: making introduction - sending INTRODUCE1",
            &self.hsid,
            rend_pt.as_inner(),
            intro_index,
        );
        intro_circ
            .m_start_conversation_last_hop(Some(intro1_real.into()), handler)
            .await
            .map_err(handle_intro_proto_error)?;
        // Status is checked by `.success()`, and we don't look at the extensions;
        // just discard the known-successful `IntroduceAck`
        let _: IntroduceAck = intro_ack_rx
            .recv(handle_intro_proto_error)
            .await?
            .success()
            .map_err(|status| FAE::IntroductionFailed {
                status,
                intro_index,
            })?;
        debug!(
            "hs conn to {}: RPT {} IPT {}: making introduction - success",
            &self.hsid,
            rend_pt.as_inner(),
            intro_index,
        );
        // Having received INTRODUCE_ACK. we can forget about this circuit
        // (and potentially tear it down).
        drop(intro_circ);
        Ok((
            rendezvous,
            Introduced {
                handshake_state,
                marker: PhantomData,
            },
        ))
    }
    /// Attempt (once) to connect a rendezvous circuit using the given intro pt
    ///
    /// Timeouts here might be due to the IPT, RPT, service,
    /// or any of the intermediate relays.
    ///
    /// If, rather than a timeout, we actually encounter some kind of error,
    /// we'll return the appropriate `FailedAttemptError`.
    /// (Who is responsible may vary, so the `FailedAttemptError` variant will reflect that.)
    ///
    /// Does not apply a timeout
    async fn complete_rendezvous(
        &'c self,
        ipt: &UsableIntroPt<'_>,
        rendezvous: Rendezvous<'c, R, M>,
        introduced: Introduced<R, M>,
    ) -> Result<Arc<ClientCirc!(R, M)>, FAE> {
        use tor_proto::circuit::handshake;
        let rend_pt = rend_pt_identity_for_error(&rendezvous.rend_relay);
        let intro_index = ipt.intro_index;
        let handle_proto_error = |error| FAE::RendezvousCompletionCircuitError {
            error,
            intro_index,
            rend_pt: rend_pt.clone(),
        };
        debug!(
            "hs conn to {}: RPT {} IPT {}: awaiting rendezvous completion",
            &self.hsid,
            rend_pt.as_inner(),
            intro_index,
        );
        let rend2_msg: Rendezvous2 = rendezvous.rend2_rx.recv(handle_proto_error).await?;
        debug!(
            "hs conn to {}: RPT {} IPT {}: received RENDEZVOUS2",
            &self.hsid,
            rend_pt.as_inner(),
            intro_index,
        );
        // In theory would be great if we could have multiple introduction attempts in parallel
        // with similar x,X values but different IPTs.  However, our HS experts don't
        // think increasing parallelism here is important:
        //   https://gitlab.torproject.org/tpo/core/arti/-/issues/913#note_2914438
        let handshake_state = introduced.handshake_state;
        // Try to complete the cryptographic handshake.
        let keygen = handshake_state
            .client_receive_rend(rend2_msg.handshake_info())
            // If this goes wrong. either the onion service has mangled the crypto,
            // or the rendezvous point has misbehaved (that that is possible is a protocol bug),
            // or we have used the wrong handshake_state (let's assume that's not true).
            //
            // If this happens we'll go and try another RPT.
            .map_err(|error| FAE::RendezvousCompletionHandshake {
                error,
                intro_index,
                rend_pt: rend_pt.clone(),
            })?;
        let params = onion_circparams_from_netparams(self.netdir.params())
            .map_err(into_internal!("Failed to build CircParameters"))?;
        rendezvous
            .rend_circ
            .m_extend_virtual(
                handshake::RelayProtocol::HsV3,
                handshake::HandshakeRole::Initiator,
                keygen,
                params,
            )
            .await
            .map_err(into_internal!(
                "actually this is probably a 'circuit closed' error" // TODO HS
            ))?;
        debug!(
            "hs conn to {}: RPT {} IPT {}: HS circuit established",
            &self.hsid,
            rend_pt.as_inner(),
            intro_index,
        );
        Ok(rendezvous.rend_circ)
    }
    /// Helper to estimate a timeout for a complicated operation
    ///
    /// `actions` is a list of `(count, action)`, where each entry
    /// represents doing `action`, `count` times sequentially.
    ///
    /// Combines the timeout estimates and returns an overall timeout.
8
    fn estimate_timeout(&self, actions: &[(u32, TimeoutsAction)]) -> Duration {
8
        // This algorithm is, perhaps, wrong.  For uncorrelated variables, a particular
8
        // percentile estimate for a sum of random variables, is not calculated by adding the
8
        // percentile estimates of the individual variables.
8
        //
8
        // But the actual lengths of times of the operations aren't uncorrelated.
8
        // If they were *perfectly* correlated, then this addition would be correct.
8
        // It will do for now; it just might be rather longer than it ought to be.
8
        actions
8
            .iter()
16
            .map(|(count, action)| {
16
                self.circpool
16
                    .m_estimate_timeout(action)
16
                    .saturating_mul(*count)
16
            })
8
            .fold(Duration::ZERO, Duration::saturating_add)
8
    }
}
/// Mocks used for testing `connect.rs`
///
/// This is different to `MockableConnectorData`,
/// which is used to *replace* this file, when testing `state.rs`.
///
/// `MocksForConnect` provides mock facilities for *testing* this file.
//
// TODO this should probably live somewhere else, maybe tor-circmgr even?
// TODO this really ought to be made by macros or something
trait MocksForConnect<R>: Clone {
    /// HS circuit pool
    type HsCircPool: MockableCircPool<R>;
    /// A random number generator
    type Rng: rand::Rng + rand::CryptoRng;
    /// Tell tests we got this descriptor text
    fn test_got_desc(&self, _: &HsDesc) {}
    /// Tell tests we got this circuit
    fn test_got_circ(&self, _: &Arc<ClientCirc!(R, Self)>) {}
    /// Tell tests we have obtained and sorted the intros like this
    fn test_got_ipts(&self, _: &[UsableIntroPt]) {}
    /// Return a random number generator
    fn thread_rng(&self) -> Self::Rng;
}
/// Mock for `HsCircPool`
///
/// Methods start with `m_` to avoid the following problem:
/// `ClientCirc::start_conversation` (say) means
/// to use the inherent method if one exists,
/// but will use a trait method if there isn't an inherent method.
///
/// So if the inherent method is renamed, the call in the impl here
/// turns into an always-recursive call.
/// This is not detected by the compiler due to the situation being
/// complicated by futures, `#[async_trait]` etc.
/// <https://github.com/rust-lang/rust/issues/111177>
#[async_trait]
trait MockableCircPool<R> {
    /// Client circuit
    type ClientCirc: MockableClientCirc;
    async fn m_get_or_launch_specific(
        &self,
        netdir: &NetDir,
        kind: HsCircKind,
        target: impl CircTarget + Send + Sync + 'async_trait,
    ) -> tor_circmgr::Result<Arc<Self::ClientCirc>>;
    /// Client circuit
    async fn m_get_or_launch_client_rend<'a>(
        &self,
        netdir: &'a NetDir,
    ) -> tor_circmgr::Result<(Arc<Self::ClientCirc>, Relay<'a>)>;
    /// Estimate timeout
    fn m_estimate_timeout(&self, action: &TimeoutsAction) -> Duration;
}
/// Mock for `ClientCirc`
#[async_trait]
trait MockableClientCirc: Debug {
    /// Client circuit
    type DirStream: AsyncRead + AsyncWrite + Send + Unpin;
    async fn m_begin_dir_stream(self: Arc<Self>) -> tor_proto::Result<Self::DirStream>;
    /// Converse
    async fn m_start_conversation_last_hop(
        &self,
        msg: Option<AnyRelayMsg>,
        reply_handler: impl MsgHandler + Send + 'static,
    ) -> tor_proto::Result<Self::Conversation<'_>>;
    /// Conversation
    type Conversation<'r>
    where
        Self: 'r;
    /// Add a virtual hop to the circuit.
    async fn m_extend_virtual(
        &self,
        protocol: tor_proto::circuit::handshake::RelayProtocol,
        role: tor_proto::circuit::handshake::HandshakeRole,
        handshake: impl tor_proto::circuit::handshake::KeyGenerator + Send,
        params: CircParameters,
    ) -> tor_proto::Result<()>;
}
impl<R: Runtime> MocksForConnect<R> for () {
    type HsCircPool = HsCircPool<R>;
    type Rng = rand::rngs::ThreadRng;
    fn thread_rng(&self) -> Self::Rng {
        rand::rng()
    }
}
#[async_trait]
impl<R: Runtime> MockableCircPool<R> for HsCircPool<R> {
    type ClientCirc = ClientCirc;
    async fn m_get_or_launch_specific(
        &self,
        netdir: &NetDir,
        kind: HsCircKind,
        target: impl CircTarget + Send + Sync + 'async_trait,
    ) -> tor_circmgr::Result<Arc<ClientCirc>> {
        HsCircPool::get_or_launch_specific(self, netdir, kind, target).await
    }
    async fn m_get_or_launch_client_rend<'a>(
        &self,
        netdir: &'a NetDir,
    ) -> tor_circmgr::Result<(Arc<ClientCirc>, Relay<'a>)> {
        HsCircPool::get_or_launch_client_rend(self, netdir).await
    }
    fn m_estimate_timeout(&self, action: &TimeoutsAction) -> Duration {
        HsCircPool::estimate_timeout(self, action)
    }
}
#[async_trait]
impl MockableClientCirc for ClientCirc {
    /// Client circuit
    type DirStream = tor_proto::stream::DataStream;
    async fn m_begin_dir_stream(self: Arc<Self>) -> tor_proto::Result<Self::DirStream> {
        ClientCirc::begin_dir_stream(self).await
    }
    async fn m_start_conversation_last_hop(
        &self,
        msg: Option<AnyRelayMsg>,
        reply_handler: impl MsgHandler + Send + 'static,
    ) -> tor_proto::Result<Self::Conversation<'_>> {
        let last_hop = self.last_hop_num()?;
        ClientCirc::start_conversation(self, msg, reply_handler, last_hop).await
    }
    type Conversation<'r> = tor_proto::circuit::Conversation<'r>;
    async fn m_extend_virtual(
        &self,
        protocol: tor_proto::circuit::handshake::RelayProtocol,
        role: tor_proto::circuit::handshake::HandshakeRole,
        handshake: impl tor_proto::circuit::handshake::KeyGenerator + Send,
        params: CircParameters,
    ) -> tor_proto::Result<()> {
        ClientCirc::extend_virtual(self, protocol, role, handshake, params).await
    }
}
#[async_trait]
impl MockableConnectorData for Data {
    type ClientCirc = ClientCirc;
    type MockGlobalState = ();
    async fn connect<R: Runtime>(
        connector: &HsClientConnector<R>,
        netdir: Arc<NetDir>,
        config: Arc<Config>,
        hsid: HsId,
        data: &mut Self,
        secret_keys: HsClientSecretKeys,
    ) -> Result<Arc<Self::ClientCirc>, ConnError> {
        connect(connector, netdir, config, hsid, data, secret_keys).await
    }
    fn circuit_is_ok(circuit: &Self::ClientCirc) -> bool {
        !circuit.is_closing()
    }
}
#[cfg(test)]
mod test {
    // @@ begin test lint list maintained by maint/add_warning @@
    #![allow(clippy::bool_assert_comparison)]
    #![allow(clippy::clone_on_copy)]
    #![allow(clippy::dbg_macro)]
    #![allow(clippy::mixed_attributes_style)]
    #![allow(clippy::print_stderr)]
    #![allow(clippy::print_stdout)]
    #![allow(clippy::single_char_pattern)]
    #![allow(clippy::unwrap_used)]
    #![allow(clippy::unchecked_duration_subtraction)]
    #![allow(clippy::useless_vec)]
    #![allow(clippy::needless_pass_by_value)]
    //! <!-- @@ end test lint list maintained by maint/add_warning @@ -->
    #![allow(dead_code, unused_variables)] // TODO HS TESTS delete, after tests are completed
    use super::*;
    use crate::*;
    use futures::FutureExt as _;
    use std::{iter, panic::AssertUnwindSafe};
    use tokio_crate as tokio;
    use tor_async_utils::JoinReadWrite;
    use tor_basic_utils::test_rng::{testing_rng, TestingRng};
    use tor_hscrypto::pk::{HsClientDescEncKey, HsClientDescEncKeypair};
    use tor_llcrypto::pk::curve25519;
    use tor_netdoc::doc::{hsdesc::test_data, netstatus::Lifetime};
    use tor_rtcompat::tokio::TokioNativeTlsRuntime;
    use tor_rtcompat::RuntimeSubstExt as _;
    #[allow(deprecated)] // TODO #1885
    use tor_rtmock::time::MockSleepProvider;
    use tracing_test::traced_test;
    #[derive(Debug, Default)]
    struct MocksGlobal {
        hsdirs_asked: Vec<OwnedCircTarget>,
        got_desc: Option<HsDesc>,
    }
    #[derive(Clone, Debug)]
    struct Mocks<I> {
        mglobal: Arc<Mutex<MocksGlobal>>,
        id: I,
    }
    impl<I> Mocks<I> {
        fn map_id<J>(&self, f: impl FnOnce(&I) -> J) -> Mocks<J> {
            Mocks {
                mglobal: self.mglobal.clone(),
                id: f(&self.id),
            }
        }
    }
    impl<R: Runtime> MocksForConnect<R> for Mocks<()> {
        type HsCircPool = Mocks<()>;
        type Rng = TestingRng;
        fn test_got_desc(&self, desc: &HsDesc) {
            self.mglobal.lock().unwrap().got_desc = Some(desc.clone());
        }
        fn test_got_ipts(&self, desc: &[UsableIntroPt]) {}
        fn thread_rng(&self) -> Self::Rng {
            testing_rng()
        }
    }
    #[allow(clippy::diverging_sub_expression)] // async_trait + todo!()
    #[async_trait]
    impl<R: Runtime> MockableCircPool<R> for Mocks<()> {
        type ClientCirc = Mocks<()>;
        async fn m_get_or_launch_specific(
            &self,
            _netdir: &NetDir,
            kind: HsCircKind,
            target: impl CircTarget + Send + Sync + 'async_trait,
        ) -> tor_circmgr::Result<Arc<Self::ClientCirc>> {
            assert_eq!(kind, HsCircKind::ClientHsDir);
            let target = OwnedCircTarget::from_circ_target(&target);
            self.mglobal.lock().unwrap().hsdirs_asked.push(target);
            // Adding the `Arc` here is a little ugly, but that's what we get
            // for using the same Mocks for everything.
            Ok(Arc::new(self.clone()))
        }
        /// Client circuit
        async fn m_get_or_launch_client_rend<'a>(
            &self,
            netdir: &'a NetDir,
        ) -> tor_circmgr::Result<(Arc<ClientCirc!(R, Self)>, Relay<'a>)> {
            todo!()
        }
        fn m_estimate_timeout(&self, action: &TimeoutsAction) -> Duration {
            Duration::from_secs(10)
        }
    }
    #[allow(clippy::diverging_sub_expression)] // async_trait + todo!()
    #[async_trait]
    impl MockableClientCirc for Mocks<()> {
        type DirStream = JoinReadWrite<futures::io::Cursor<Box<[u8]>>, futures::io::Sink>;
        type Conversation<'r> = &'r ();
        async fn m_begin_dir_stream(self: Arc<Self>) -> tor_proto::Result<Self::DirStream> {
            let response = format!(
                r#"HTTP/1.1 200 OK
{}"#,
                test_data::TEST_DATA_2
            )
            .into_bytes()
            .into_boxed_slice();
            Ok(JoinReadWrite::new(
                futures::io::Cursor::new(response),
                futures::io::sink(),
            ))
        }
        async fn m_start_conversation_last_hop(
            &self,
            msg: Option<AnyRelayMsg>,
            reply_handler: impl MsgHandler + Send + 'static,
        ) -> tor_proto::Result<Self::Conversation<'_>> {
            todo!()
        }
        async fn m_extend_virtual(
            &self,
            protocol: tor_proto::circuit::handshake::RelayProtocol,
            role: tor_proto::circuit::handshake::HandshakeRole,
            handshake: impl tor_proto::circuit::handshake::KeyGenerator + Send,
            params: CircParameters,
        ) -> tor_proto::Result<()> {
            todo!()
        }
    }
    #[traced_test]
    #[tokio::test]
    async fn test_connect() {
        let valid_after = humantime::parse_rfc3339("2023-02-09T12:00:00Z").unwrap();
        let fresh_until = valid_after + humantime::parse_duration("1 hours").unwrap();
        let valid_until = valid_after + humantime::parse_duration("24 hours").unwrap();
        let lifetime = Lifetime::new(valid_after, fresh_until, valid_until).unwrap();
        let netdir = tor_netdir::testnet::construct_custom_netdir_with_params(
            tor_netdir::testnet::simple_net_func,
            iter::empty::<(&str, _)>(),
            Some(lifetime),
        )
        .expect("failed to build default testing netdir");
        let netdir = Arc::new(netdir.unwrap_if_sufficient().unwrap());
        let runtime = TokioNativeTlsRuntime::current().unwrap();
        let now = humantime::parse_rfc3339("2023-02-09T12:00:00Z").unwrap();
        #[allow(deprecated)] // TODO #1885
        let mock_sp = MockSleepProvider::new(now);
        let runtime = runtime
            .with_sleep_provider(mock_sp.clone())
            .with_coarse_time_provider(mock_sp);
        let time_period = netdir.hs_time_period();
        let mglobal = Arc::new(Mutex::new(MocksGlobal::default()));
        let mocks = Mocks { mglobal, id: () };
        // From C Tor src/test/test_hs_common.c test_build_address
        let hsid = test_data::TEST_HSID_2.into();
        let mut data = Data::default();
        let pk: HsClientDescEncKey = curve25519::PublicKey::from(test_data::TEST_PUBKEY_2).into();
        let sk = curve25519::StaticSecret::from(test_data::TEST_SECKEY_2).into();
        let mut secret_keys_builder = HsClientSecretKeysBuilder::default();
        secret_keys_builder.ks_hsc_desc_enc(HsClientDescEncKeypair::new(pk.clone(), sk));
        let secret_keys = secret_keys_builder.build().unwrap();
        let ctx = Context::new(
            &runtime,
            &mocks,
            netdir,
            Default::default(),
            hsid,
            secret_keys,
            mocks.clone(),
        )
        .unwrap();
        let _got = AssertUnwindSafe(ctx.connect(&mut data))
            .catch_unwind() // TODO HS TESTS: remove this and the AssertUnwindSafe
            .await;
        let (hs_blind_id_key, subcredential) = HsIdKey::try_from(hsid)
            .unwrap()
            .compute_blinded_key(time_period)
            .unwrap();
        let hs_blind_id = hs_blind_id_key.id();
        let sk = curve25519::StaticSecret::from(test_data::TEST_SECKEY_2).into();
        let hsdesc = HsDesc::parse_decrypt_validate(
            test_data::TEST_DATA_2,
            &hs_blind_id,
            now,
            &subcredential,
            Some(&HsClientDescEncKeypair::new(pk, sk)),
        )
        .unwrap()
        .dangerously_assume_timely();
        let mglobal = mocks.mglobal.lock().unwrap();
        assert_eq!(mglobal.hsdirs_asked.len(), 1);
        // TODO hs: here and in other places, consider implementing PartialEq instead, or creating
        // an assert_dbg_eq macro (which would be part of a test_helpers crate or something)
        assert_eq!(
            format!("{:?}", mglobal.got_desc),
            format!("{:?}", Some(hsdesc))
        );
        // Check how long the descriptor is valid for
        let (start_time, end_time) = data.desc.as_ref().unwrap().bounds();
        assert_eq!(start_time, None);
        let desc_valid_until = humantime::parse_rfc3339("2023-02-11T20:00:00Z").unwrap();
        assert_eq!(end_time, Some(desc_valid_until));
        // TODO HS TESTS: check the circuit in got is the one we gave out
        // TODO HS TESTS: continue with this
    }
    // TODO HS TESTS: Test IPT state management and expiry:
    //   - obtain a test descriptor with only a broken ipt
    //     (broken in the sense that intro can be attempted, but will fail somehow)
    //   - try to make a connection and expect it to fail
    //   - assert that the ipt data isn't empty
    //   - cause the descriptor to expire (advance clock)
    //   - start using a mocked RNG if we weren't already and pin its seed here
    //   - make a new descriptor with two IPTs: the broken one from earlier, and a new one
    //   - make a new connection
    //   - use test_got_ipts to check that the random numbers
    //     would sort the bad intro first, *and* that the good one is appears first
    //   - assert that connection succeeded
    //   - cause the circuit and descriptor to expire (advance clock)
    //   - go back to the previous descriptor contents, but with a new validity period
    //   - try to make a connection
    //   - use test_got_ipts to check that only the broken ipt is present
    // TODO HS TESTS: test retries (of every retry loop we have here)
    // TODO HS TESTS: test error paths
}