/* ======================================================================== * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Collections.Concurrent; using System.Linq; using System.Threading; using System.Threading.Tasks; using Moq; using NUnit.Framework; using Opc.Ua.Client.Tests.Stack.Client.Fakes; using static Opc.Ua.Client.Tests.Stack.Client.Fakes.ManagedSessionReconnectHarness; namespace Opc.Ua.Client.Tests.ManagedSession { /// /// Exercises public manual channel replacement without bypassing managed-session or channel-manager ownership. /// [TestFixture] [Category("Client")] [NonParallelizable] public sealed class ManagedSessionChannelReplacementTests { /// /// A supplied owning lease becomes the current binding and the wrapper follows only its recovery events. /// Retired final notifications cannot stop its keepalive or release its active recovery. /// /// Whether to supply a fresh lease from another real channel manager. [TestCase(false, TestName = "SupplyingCurrentManagedLeasePreservesBindingAndEventsAsync")] [TestCase(true, TestName = "SuppliedManagedLeaseRebindsSessionAndChannelEventsAsync")] public async Task SuppliedManagedLeaseRebindsSessionAndChannelEventsAsync(bool replaceChannel) { var replacementReconnect = new AsyncOperationGate(); await using var replacementChannels = new SessionChannelHarness( timeProvider: new ObservableFakeTimeProvider(), configureChannel: channel => { channel.SupportedFeatures = TransportChannelFeatures.Reconnect; channel.ReconnectHandler = replacementReconnect.WaitAsync; }); await using var harness = new ManagedSessionReconnectHarness(TimeSpan.Zero) { RecoveryActivationStatus = StatusCodes.Good }; ConfigureManualReconnectPolicy(harness); await harness.ConnectAsync(false).ConfigureAwait(false); harness.RecoveryActivation.Release(); Session originalSession = harness.Session.InnerSession; IManagedTransportChannel originalChannel = harness.Lease; NodeId originalSessionId = originalSession.SessionId; var sibling = new Mock(); sibling.SetupGet(participant => participant.Id).Returns("retired-channel-participant"); sibling.SetupGet(participant => participant.Endpoint).Returns(originalSession.ConfiguredEndpoint); sibling.Setup(participant => participant.OnReconnectAsync( It.IsAny(), It.IsAny(), It.IsAny())) .Returns(new ValueTask(ParticipantReconnectResult.Reactivated)); using IManagedTransportChannel retiredSibling = await harness.Channels.Manager.GetAsync( sibling.Object, harness.CancellationToken).ConfigureAwait(false); using IManagedTransportChannel? replacement = replaceChannel ? await replacementChannels.Manager.GetAsync(originalSession, harness.CancellationToken) .ConfigureAwait(false) : null; IManagedTransportChannel suppliedChannel = replacement ?? originalChannel; AsyncOperationGate reconnectGate = replaceChannel ? replacementReconnect : harness.TransportReconnect; var forwarded = new ConcurrentQueue(); Task? channelRecovery = null; try { await WaitForPhaseAsync( harness.Session.ReconnectAsync(null, suppliedChannel, harness.CancellationToken), "manual reconnect with the supplied owning lease").ConfigureAwait(false); IManagedTransportChannel? bindingAfterReplacement = originalSession.ManagedChannel; IClientChannelManager? managerAfterReplacement = originalSession.ChannelManager; harness.Session.ChannelStateChanged += (_, change) => forwarded.Enqueue(change); channelRecovery = suppliedChannel.Manager.ReconnectAsync( suppliedChannel, harness.CancellationToken).AsTask(); await WaitForPhaseAsync(reconnectGate.Entered, "current channel transport recovery") .ConfigureAwait(false); bool ownerDuringRecovery = originalSession.ChannelRecoveryInProgress; ChannelStateChange[] eventsDuringRecovery = [.. forwarded]; Assert.That(suppliedChannel.State, Is.EqualTo(ChannelState.TransportReconnecting)); Assert.That(channelRecovery.IsCompleted, Is.False); Assert.That(harness.Session.StateMachine.State, Is.EqualTo(ConnectionState.Connected)); if (replaceChannel) { ParticipantReconnectResult retiredResult = await ((IReconnectParticipant)originalSession) .OnReconnectAsync(originalChannel, -1, harness.CancellationToken).AsTask() .WaitAsync(PhaseTimeout).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(retiredResult, Is.EqualTo(ParticipantReconnectResult.FatalForParticipant)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.True); Assert.That(originalSession.ManagedChannel, Is.SameAs(suppliedChannel)); Assert.That(suppliedChannel.State, Is.EqualTo(ChannelState.TransportReconnecting)); Assert.That(channelRecovery.IsCompleted, Is.False); Assert.That(reconnectGate.Cancelled.IsCompleted, Is.False); }); } reconnectGate.Release(); harness.RecoveryActivation.Release(); await WaitForPhaseAsync(channelRecovery, "current channel recovery completion").ConfigureAwait(false); ChannelStateChange[] currentEvents = [.. forwarded]; if (replaceChannel) { harness.TransportReconnect.Release(); await WaitForPhaseAsync( harness.Channels.Manager.ReconnectAsync(retiredSibling, harness.CancellationToken).AsTask(), "independent recovery of the retired entry").ConfigureAwait(false); await AssertRetiredFinalPreservesKeepAliveAsync(harness, originalSession, originalChannel) .ConfigureAwait(false); } DataValue value = await harness.Session.ReadValueAsync( VariableIds.Server_ServerStatus_State, harness.CancellationToken).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(harness.Session.InnerSession, Is.SameAs(originalSession)); Assert.That(originalSession.SessionId, Is.EqualTo(originalSessionId)); Assert.That(harness.Lease, Is.SameAs(suppliedChannel)); Assert.That(bindingAfterReplacement, Is.SameAs(suppliedChannel)); Assert.That(managerAfterReplacement, Is.SameAs(suppliedChannel.Manager)); Assert.That(ownerDuringRecovery, Is.True); Assert.That(eventsDuringRecovery.Select(change => change.NewState), Is.EqualTo([ChannelState.TransportReconnecting])); Assert.That(currentEvents.Select(change => change.NewState), Is.EqualTo(s_recoveryStates)); Assert.That(forwarded.ToArray(), Is.EqualTo(currentEvents), "A retired entry must not publish channel events through the current managed session."); Assert.That(originalSession.ManagedChannel, Is.SameAs(suppliedChannel)); Assert.That(originalSession.ChannelManager, Is.SameAs(suppliedChannel.Manager)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.False); Assert.That(originalSession.Reconnecting, Is.False); Assert.That(value.StatusCode, Is.EqualTo(StatusCodes.Good)); Assert.That(value.WrappedValue.TryGetValue(out int state), Is.True); Assert.That(state, Is.EqualTo((int)ServerState.Running)); }); } finally { replacementReconnect.Release(); harness.ReleaseAllGates(); if (channelRecovery != null) { await WaitForPhaseAsync(channelRecovery, "current channel recovery cleanup").ConfigureAwait(false); } } } /// /// Failure after activation starts keeps the managed binding truthful and permits recovery of the live session. /// /// Whether caller cancellation replaces the scripted activation failure. [TestCase(false, TestName = "FailedSuppliedReconnectKeepsBindingRecoverableAsync")] [TestCase(true, TestName = "CancelledSuppliedReconnectKeepsBindingRecoverableAsync")] public async Task UnsuccessfulSuppliedReconnectKeepsBindingRecoverableAsync(bool cancelActivation) { var activation = new AsyncOperationGate(); int activationCount = 0; await using var replacementChannels = new SessionChannelHarness( timeProvider: new ObservableFakeTimeProvider(), configureChannel: channel => { channel.SupportedFeatures = TransportChannelFeatures.Reconnect; channel.RequestHandler = async (request, ct) => { if (request is ActivateSessionRequest && Interlocked.Increment(ref activationCount) == 1) { await activation.WaitAsync(ct).ConfigureAwait(false); throw new ServiceResultException(StatusCodes.BadUserAccessDenied); } return channel.CreateResponse(request); }; }); await using var harness = new ManagedSessionReconnectHarness(TimeSpan.Zero) { RecoveryActivationStatus = StatusCodes.Good }; ConfigureManualReconnectPolicy(harness); await harness.ConnectAsync(false).ConfigureAwait(false); harness.ReleaseAllGates(); Session originalSession = harness.Session.InnerSession; NodeId originalSessionId = originalSession.SessionId; IManagedTransportChannel suppliedChannel = await replacementChannels.Manager.GetAsync( originalSession, harness.CancellationToken).ConfigureAwait(false); using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(harness.CancellationToken); var disconnected = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); harness.Session.ConnectionStateChanged += (_, change) => { if (change.NewState == ConnectionState.Disconnected) { disconnected.TrySetResult(true); } }; try { await CompleteUnsuccessfulReconnectAsync( harness.Session.ReconnectAsync(null, suppliedChannel, cancellation.Token), activation, cancellation, cancelActivation).ConfigureAwait(false); await WaitForPhaseAsync(disconnected.Task, "manual reconnect attempt fully settled") .ConfigureAwait(false); IManagedTransportChannel actualAfterFailure = harness.Lease; IManagedTransportChannel? bindingAfterFailure = originalSession.ManagedChannel; IClientChannelManager? managerAfterFailure = originalSession.ChannelManager; bool recoveryAfterFailure = originalSession.ChannelRecoveryInProgress; bool reconnectingAfterFailure = originalSession.Reconnecting; ScriptedChannel recoveryTransport = ReferenceEquals(actualAfterFailure, suppliedChannel) ? replacementChannels.CreatedChannels[0] : harness.Channels.CreatedChannels[0]; int reconnectsBeforeRetry = recoveryTransport.ReconnectCount; var forwarded = new ConcurrentQueue(); harness.Session.ChannelStateChanged += (_, change) => forwarded.Enqueue(change); await WaitForPhaseAsync( harness.Session.ReconnectAsync(null, null, harness.CancellationToken), "subsequent recovery after unsuccessful manual activation").ConfigureAwait(false); DataValue value = await harness.Session.ReadValueAsync( VariableIds.Server_ServerStatus_State, harness.CancellationToken).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(harness.Session.InnerSession, Is.SameAs(originalSession)); Assert.That(originalSession.SessionId, Is.EqualTo(originalSessionId)); Assert.That(bindingAfterFailure, Is.SameAs(actualAfterFailure)); Assert.That(managerAfterFailure, Is.SameAs(actualAfterFailure.Manager)); Assert.That(recoveryAfterFailure, Is.False); Assert.That(reconnectingAfterFailure, Is.False); Assert.That(harness.Lease, Is.SameAs(actualAfterFailure), "A successful retry must reconnect the transport that the failed attempt actually retained."); Assert.That(recoveryTransport.ReconnectCount, Is.EqualTo(reconnectsBeforeRetry + 1)); Assert.That(forwarded.Select(change => change.NewState), Is.EqualTo(s_recoveryStates)); Assert.That(originalSession.ManagedChannel, Is.SameAs(harness.Lease)); Assert.That(originalSession.ChannelManager, Is.SameAs(harness.Lease.Manager)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.False); Assert.That(originalSession.Reconnecting, Is.False); Assert.That(harness.Session.StateMachine.State, Is.EqualTo(ConnectionState.Connected)); Assert.That(value.StatusCode, Is.EqualTo(StatusCodes.Good)); Assert.That(value.WrappedValue.TryGetValue(out int state), Is.True); Assert.That(state, Is.EqualTo((int)ServerState.Running)); }); } finally { activation.Release(); await suppliedChannel.CloseAsync(harness.CancellationToken).ConfigureAwait(false); } } /// /// A competing public Session reconnect cannot release or replace an active channel-reactivation owner. /// [Test] public async Task SuppliedReconnectDuringActivationPreservesRecoveryOwnerAsync() { await using var replacementChannels = new SessionChannelHarness( timeProvider: new ObservableFakeTimeProvider(), configureChannel: channel => channel.SupportedFeatures = TransportChannelFeatures.Reconnect); await using var harness = new ManagedSessionReconnectHarness(TimeSpan.Zero) { RecoveryActivationStatus = StatusCodes.Good }; await harness.ConnectAsync(false).ConfigureAwait(false); Session originalSession = harness.Session.InnerSession; IManagedTransportChannel originalChannel = harness.Lease; NodeId originalSessionId = originalSession.SessionId; IManagedTransportChannel suppliedChannel = await replacementChannels.Manager.GetAsync( originalSession, harness.CancellationToken).ConfigureAwait(false); try { ScriptedChannel replacementTransport = replacementChannels.CreatedChannels[0]; int overlappingActivation = 0; replacementTransport.RequestHandler = (request, _) => { if (request is ActivateSessionRequest && !harness.RecoveryActivation.Exited.IsCompleted) { Interlocked.Increment(ref overlappingActivation); } return new ValueTask(replacementTransport.CreateResponse(request)); }; harness.TransportReconnect.Release(); Task recovery = harness.StartRecoveryAsync(); await WaitForPhaseAsync(harness.RecoveryActivation.Entered, "active manager-owned reactivation") .ConfigureAwait(false); await ObserveCompetingReconnectAsync( originalSession.ReconnectAsync(null, suppliedChannel, harness.CancellationToken), recovery, () => Assert.Multiple(() => { Assert.That(originalSession.Reconnecting, Is.True); Assert.That(originalSession.ChannelRecoveryInProgress, Is.True); Assert.That(originalSession.ManagedChannel, Is.SameAs(originalChannel)); Assert.That(originalSession.TransportChannel, Is.SameAs(originalChannel)); Assert.That(originalChannel.State, Is.EqualTo(ChannelState.TransportConnectedSessionReactivating)); Assert.That(harness.RecoveryActivation.Exited.IsCompleted, Is.False); Assert.That(harness.RecoveryActivation.Cancelled.IsCompleted, Is.False); Assert.That(recovery.IsCompleted, Is.False); Assert.That(Volatile.Read(ref overlappingActivation), Is.Zero); }), harness.ReleaseAllGates).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(Volatile.Read(ref overlappingActivation), Is.Zero); Assert.That(harness.Session.InnerSession, Is.SameAs(originalSession)); Assert.That(originalSession.SessionId, Is.EqualTo(originalSessionId)); Assert.That(originalSession.ManagedChannel, Is.SameAs(harness.Lease)); Assert.That(originalSession.ChannelManager, Is.SameAs(harness.Lease.Manager)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.False); Assert.That(originalSession.Reconnecting, Is.False); }); } finally { harness.ReleaseAllGates(); await suppliedChannel.CloseAsync(harness.CancellationToken).ConfigureAwait(false); } } /// /// A competing supplied-channel reconnect cannot replace an active subscription-restoration owner. /// /// Whether the outer managed session owns deferred subscription restoration. /// /// Whether to request replacement through the public managed-session wrapper. /// [TestCase(false, false, TestName = "SuppliedReconnectDuringRestorationPreservesChannelOwnerAsync")] [TestCase(false, true, TestName = "ManagedSuppliedReconnectDuringRestorationPreservesChannelOwnerAsync")] [TestCase(true, true, TestName = "SuppliedReconnectDuringRestorationPreservesOuterOwnerAsync")] public async Task SuppliedReconnectDuringRestorationPreservesOwnerAsync( bool outerOwner, bool useManagedSession) { await using var replacementChannels = new SessionChannelHarness( timeProvider: new ObservableFakeTimeProvider(), configureChannel: channel => channel.SupportedFeatures = TransportChannelFeatures.Reconnect); await using var harness = new ManagedSessionReconnectHarness(TimeSpan.Zero) { HoldSubscriptionRestoration = true }; ConfigureManualReconnectPolicy(harness); await harness.ConnectAsync(false).ConfigureAwait(false); Session originalSession = harness.Session.InnerSession; IManagedTransportChannel originalChannel = harness.Lease; harness.AddSubscription(); await WaitForPhaseAsync(harness.NextMonitoredItemsAsync().AsTask(), "initial monitored item") .ConfigureAwait(false); IManagedTransportChannel suppliedChannel = await replacementChannels.Manager.GetAsync( originalSession, harness.CancellationToken).ConfigureAwait(false); try { ScriptedChannel replacementTransport = replacementChannels.CreatedChannels[0]; ScriptedChannel originalTransport = harness.Channels.CreatedChannels[0]; int overlappingActivation = 0; replacementTransport.RequestHandler = (request, ct) => { if (request is ActivateSessionRequest && !harness.ReplacementSubscription.Exited.IsCompleted) { Interlocked.Increment(ref overlappingActivation); } return originalTransport.RequestHandler!(request, ct); }; harness.TransportReconnect.Release(); harness.RecoveryActivation.Release(); harness.ReplacementSession.Release(); Task recovery = outerOwner ? harness.StartOuterRecoveryAsync() : harness.StartRecoveryAsync(); await WaitForPhaseAsync( harness.ReplacementSubscription.Entered, "active subscription restoration") .ConfigureAwait(false); await ObserveCompetingReconnectAsync( useManagedSession ? harness.Session.ReconnectAsync(null, suppliedChannel, harness.CancellationToken) : originalSession.ReconnectAsync(null, suppliedChannel, harness.CancellationToken), recovery, () => Assert.Multiple(() => { Assert.That(harness.Session.StateMachine.State, Is.EqualTo(outerOwner ? ConnectionState.Reconnecting : ConnectionState.Connected)); Assert.That(originalSession.Reconnecting, Is.False); if (!outerOwner) { Assert.That(originalSession.ChannelRecoveryInProgress, Is.True); } Assert.That(originalSession.ManagedChannel, Is.SameAs(originalChannel)); Assert.That(originalSession.TransportChannel, Is.SameAs(originalChannel)); Assert.That(originalSession.ChannelManager, Is.SameAs(harness.Channels.Manager)); Assert.That(harness.ReplacementSubscription.Exited.IsCompleted, Is.False); Assert.That(harness.ReplacementSubscription.Cancelled.IsCompleted, Is.False); Assert.That(recovery.IsCompleted, Is.False); Assert.That(Volatile.Read(ref overlappingActivation), Is.Zero); }), harness.ReleaseAllGates).ConfigureAwait(false); CreateMonitoredItemsRequest restored = await harness.NextMonitoredItemsAsync().AsTask() .WaitAsync(PhaseTimeout).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(Volatile.Read(ref overlappingActivation), Is.Zero); Assert.That(harness.Session.InnerSession, Is.SameAs(originalSession)); Assert.That(originalSession.SessionId, Is.EqualTo(new NodeId("session-2", 1))); Assert.That(originalSession.ManagedChannel, Is.SameAs(harness.Lease)); Assert.That(originalSession.ChannelManager, Is.SameAs(harness.Lease.Manager)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.False); Assert.That(originalSession.Reconnecting, Is.False); Assert.That(harness.Session.StateMachine.State, Is.EqualTo(ConnectionState.Connected)); Assert.That(restored.RequestHeader.AuthenticationToken, Is.EqualTo(new NodeId("token-2", 1))); Assert.That(restored.ItemsToCreate.Count, Is.EqualTo(1)); }); } finally { harness.ReleaseAllGates(); await suppliedChannel.CloseAsync(harness.CancellationToken).ConfigureAwait(false); } } /// /// A late activation response from a cancelled cycle cannot clear ownership of recovery on a supplied lease. /// [Test] public async Task LateRetiredActivationCannotClearReplacementRecoveryOwnerAsync() { var successorTransport = new AsyncOperationGate(); await using var replacementChannels = new SessionChannelHarness( timeProvider: new ObservableFakeTimeProvider(), configureChannel: channel => { channel.SupportedFeatures = TransportChannelFeatures.Reconnect; channel.ReconnectHandler = successorTransport.WaitAsync; }); await using var harness = new ManagedSessionReconnectHarness(TimeSpan.Zero) { RecoveryActivationStatus = StatusCodes.Good }; ConfigureManualReconnectPolicy(harness); harness.RecoveryActivation.IgnoreCancellation = true; await harness.ConnectAsync(false).ConfigureAwait(false); Session originalSession = harness.Session.InnerSession; IManagedTransportChannel originalChannel = harness.Lease; NodeId originalSessionId = originalSession.SessionId; ScriptedChannel originalTransport = harness.Channels.CreatedChannels[0]; await WaitForPhaseAsync( originalTransport.NextOperationAsync(harness.CancellationToken).AsTask(), "initial session activation").ConfigureAwait(false); harness.TransportReconnect.Release(); Task retiredRecovery = harness.StartRecoveryAsync(); Task? successorRecovery = null; try { await WaitForPhaseAsync(harness.RecoveryActivation.Entered, "old activation awaiting its response") .ConfigureAwait(false); ScriptedChannel.RequestOperation lateActivation = await originalTransport .NextOperationAsync(harness.CancellationToken).AsTask() .WaitAsync(PhaseTimeout).ConfigureAwait(false); await WaitForPhaseAsync(originalChannel.CloseAsync(harness.CancellationToken).AsTask(), "retiring the old owning lease").ConfigureAwait(false); await WaitForPhaseAsync(harness.RecoveryActivation.Cancelled, "old activation token cancellation") .ConfigureAwait(false); Assert.That(async () => await retiredRecovery.WaitAsync(PhaseTimeout).ConfigureAwait(false), Throws.InstanceOf()); Assert.Multiple(() => { Assert.That(originalChannel.State, Is.EqualTo(ChannelState.Closed)); Assert.That(originalSession.Reconnecting, Is.False); Assert.That(harness.RecoveryActivation.Exited.IsCompleted, Is.False); Assert.That(lateActivation.Operation.IsCompleted, Is.False); }); using IManagedTransportChannel suppliedChannel = await replacementChannels.Manager.GetAsync( originalSession, harness.CancellationToken).ConfigureAwait(false); await WaitForPhaseAsync( harness.Session.ReconnectAsync(null, suppliedChannel, harness.CancellationToken), "manual replacement while the retired raw response remains pending").ConfigureAwait(false); var forwarded = new ConcurrentQueue(); harness.Session.ChannelStateChanged += (_, change) => forwarded.Enqueue(change); successorRecovery = replacementChannels.Manager.ReconnectAsync( suppliedChannel, harness.CancellationToken).AsTask(); await WaitForPhaseAsync(successorTransport.Entered, "new lease recovery holding transport reconnect") .ConfigureAwait(false); Assert.That(originalSession.ChannelRecoveryInProgress, Is.True); harness.RecoveryActivation.Release(); IServiceResponse lateResponse = await lateActivation.Operation.WaitAsync(PhaseTimeout) .ConfigureAwait(false); await WaitForPhaseAsync(harness.RecoveryActivation.Exited, "late retired activation response") .ConfigureAwait(false); Assert.Multiple(() => { Assert.That(lateResponse.ResponseHeader.ServiceResult, Is.EqualTo(StatusCodes.Good)); Assert.That(harness.Session.InnerSession, Is.SameAs(originalSession)); Assert.That(originalSession.SessionId, Is.EqualTo(originalSessionId)); Assert.That(originalSession.ManagedChannel, Is.SameAs(suppliedChannel)); Assert.That(originalSession.ChannelManager, Is.SameAs(replacementChannels.Manager)); Assert.That(harness.Lease, Is.SameAs(suppliedChannel)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.True); Assert.That(suppliedChannel.State, Is.EqualTo(ChannelState.TransportReconnecting)); Assert.That(successorRecovery.IsCompleted, Is.False); Assert.That(successorTransport.Cancelled.IsCompleted, Is.False); Assert.That(harness.Session.StateMachine.State, Is.EqualTo(ConnectionState.Connected)); Assert.That(forwarded.Select(change => change.NewState), Is.EqualTo([ChannelState.TransportReconnecting])); }); successorTransport.Release(); await WaitForPhaseAsync(successorRecovery, "successor recovery completing after the late response") .ConfigureAwait(false); Assert.Multiple(() => { Assert.That(originalSession.ManagedChannel, Is.SameAs(suppliedChannel)); Assert.That(originalSession.ChannelRecoveryInProgress, Is.False); Assert.That(originalSession.Reconnecting, Is.False); Assert.That(suppliedChannel.State, Is.EqualTo(ChannelState.Ready)); Assert.That(forwarded.Select(change => change.NewState), Is.EqualTo(s_recoveryStates)); }); } finally { successorTransport.Release(); harness.ReleaseAllGates(); if (successorRecovery != null) { await WaitForPhaseAsync(successorRecovery, "successor recovery cleanup").ConfigureAwait(false); } } } private static async Task AssertRetiredFinalPreservesKeepAliveAsync( ManagedSessionReconnectHarness harness, Session session, IManagedTransportChannel retiredChannel) { var keepAlives = System.Threading.Channels.Channel.CreateUnbounded(); harness.Session.KeepAlive += (_, args) => keepAlives.Writer.TryWrite(args); var interval = TimeSpan.FromMilliseconds(100); Task timerChanged = harness.Clock.WaitForTimerChangedAsync(interval, interval); harness.Session.KeepAliveInterval = (int)interval.TotalMilliseconds; await WaitForPhaseAsync(timerChanged, "replacement keepalive schedule").ConfigureAwait(false); harness.Clock.Advance(interval); KeepAliveEventArgs before = await keepAlives.Reader.ReadAsync(harness.CancellationToken).AsTask() .WaitAsync(PhaseTimeout).ConfigureAwait(false); DateTime lastKeepAlive = session.LastKeepAliveTime; IReconnectParticipant participant = session; using var cancellation = new CancellationTokenSource(); await cancellation.CancelAsync().ConfigureAwait(false); Assert.That(() => participant.OnReconnectAsync(retiredChannel, -1, cancellation.Token) .AsTask().WaitAsync(PhaseTimeout), Throws.InstanceOf()); ParticipantReconnectResult result = await participant .OnReconnectAsync(retiredChannel, -1, harness.CancellationToken).AsTask() .WaitAsync(PhaseTimeout).ConfigureAwait(false); harness.Clock.Advance(interval); KeepAliveEventArgs after = await keepAlives.Reader.ReadAsync(harness.CancellationToken).AsTask() .WaitAsync(PhaseTimeout).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(result, Is.EqualTo(ParticipantReconnectResult.FatalForParticipant)); Assert.That(before.Status, Is.Null); Assert.That(before.CurrentState, Is.EqualTo(ServerState.Running)); Assert.That(after.Status, Is.Null); Assert.That(after.CurrentState, Is.EqualTo(ServerState.Running)); Assert.That(session.LastKeepAliveTime, Is.EqualTo(lastKeepAlive.Add(interval))); Assert.That(session.ManagedChannel, Is.SameAs(harness.Lease)); Assert.That(harness.Lease.State, Is.EqualTo(ChannelState.Ready)); Assert.That(harness.Session.StateMachine.State, Is.EqualTo(ConnectionState.Connected)); }); } private static void ConfigureManualReconnectPolicy(ManagedSessionReconnectHarness harness) { harness.OuterPolicy.Setup(policy => policy.GetNextDelay( It.IsAny(), It.IsAny())) .Returns(static (attempt, _) => attempt == 0 ? TimeSpan.Zero : null); } private static async Task CompleteUnsuccessfulReconnectAsync( Task reconnect, AsyncOperationGate activation, CancellationTokenSource cancellation, bool cancelActivation) { await WaitForPhaseAsync(activation.Entered, "supplied-channel activation").ConfigureAwait(false); if (cancelActivation) { await cancellation.CancelAsync().WaitAsync(PhaseTimeout).ConfigureAwait(false); await WaitForPhaseAsync(activation.Cancelled, "caller cancellation reaching activation") .ConfigureAwait(false); } else { activation.Release(); } await WaitForPhaseAsync(Task.WhenAny(reconnect), "unsuccessful manual reconnect completion") .ConfigureAwait(false); if (cancelActivation) { Exception error = Assert.CatchAsync(async () => await reconnect.ConfigureAwait(false))!; Assert.That(error, Is.InstanceOf().Or.InstanceOf()); if (error is ServiceResultException serviceError) { // The worker can settle its cancellation failure before the caller's cancellation wait wins. Assert.That(serviceError.StatusCode, Is.EqualTo(StatusCodes.Bad)); } Assert.That(cancellation.IsCancellationRequested, Is.True); Assert.That(activation.Cancelled.IsCompleted, Is.True); } else { Assert.That(async () => await reconnect.ConfigureAwait(false), Throws.InstanceOf() .With.Property(nameof(ServiceResultException.StatusCode)) .EqualTo(StatusCodes.BadUserAccessDenied)); } await WaitForPhaseAsync(activation.Exited, "unsuccessful activation unwind").ConfigureAwait(false); } private static async Task ObserveCompetingReconnectAsync( Task competingReconnect, Task recovery, Action assertOwner, Action release) { try { assertOwner(); } finally { release(); await WaitForPhaseAsync(recovery, "original recovery completion").ConfigureAwait(false); await WaitForPhaseAsync(Task.WhenAny(competingReconnect), "competing reconnect completion") .ConfigureAwait(false); try { await competingReconnect.ConfigureAwait(false); } catch (ServiceResultException exception) { Assert.That(exception.StatusCode, Is.EqualTo(StatusCodes.BadInvalidState)); } } } private static async Task WaitForPhaseAsync(Task phase, string description) { try { await phase.WaitAsync(PhaseTimeout).ConfigureAwait(false); } catch (TimeoutException exception) { throw new AssertionException($"Timed out waiting for {description}.", exception); } } private static readonly ChannelState[] s_recoveryStates = [ ChannelState.TransportReconnecting, ChannelState.TransportConnectedSessionReactivating, ChannelState.Ready ]; } }