/* ======================================================================== * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Globalization; using System.Linq; using System.Reflection; using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; using Moq; using NUnit.Framework; using Opc.Ua.Bindings; using Opc.Ua.Client.Tests.Stack.Client.Fakes; using Opc.Ua.Security.Certificates; using Opc.Ua.Tests; // CA2000: test code; many disposables are ownership-transferred to test fixtures or short-lived. #pragma warning disable CA2000 namespace Opc.Ua.Client.Tests.Stack.Client { /// /// Unit tests for the managed (sharing, refcount, coalesced /// reconnect, participant notification) behavior of /// implementing /// . /// [TestFixture] [Category("Client")] [SetCulture("en-us")] [SetUICulture("en-us")] [Parallelizable] public sealed class ClientChannelManagerManagedTests { private static readonly ICertificateFactory s_factory = DefaultCertificateFactory.Instance; /// /// How long a test waits for an asynchronous operation it has already /// unblocked to be observed as complete. /// /// /// This is a hang detector, not a latency assertion. These tests drive a /// fake clock, so the work under test finishes in microseconds; what is /// being waited on is purely the thread-pool scheduling of the /// continuation chain. On a saturated CI agent the pool injects threads /// at roughly one per second, so a short budget times out while the /// operation is merely queued - the channel has already faulted and its /// completion source has already been signalled. Keep this generous /// enough that it never fires on a healthy run, and far below the /// blame-hang timeout so a genuine deadlock still fails the job quickly. /// private static readonly TimeSpan s_completionTimeout = TimeSpan.FromSeconds(60); [Test] public void ChannelKeyEqualityIsValueBased() { using Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA(); ConfiguredEndpoint endpoint1 = GetTestEndpoint(serverCert); ConfiguredEndpoint endpoint2 = GetTestEndpoint(serverCert); var k1 = ManagedChannelKey.FromEndpoint(endpoint1); var k2 = ManagedChannelKey.FromEndpoint(endpoint2); Assert.That(k1, Is.EqualTo(k2)); Assert.That(k1.GetHashCode(), Is.EqualTo(k2.GetHashCode())); } [Test] public void ChannelKeyDistinguishesReverseFromForward() { using Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA(); ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var forward = ManagedChannelKey.FromEndpoint(endpoint); var reverse = ManagedChannelKey.FromEndpoint( endpoint, reverseConnectionIdentity: new object()); Assert.That(forward, Is.Not.EqualTo(reverse)); } [Test] public void ChannelKeyDistinguishesDifferentReverseHandles() { using Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA(); ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var r1 = ManagedChannelKey.FromEndpoint(endpoint, reverseConnectionIdentity: new object()); var r2 = ManagedChannelKey.FromEndpoint(endpoint, reverseConnectionIdentity: new object()); Assert.That(r1, Is.Not.EqualTo(r2)); } [Test] public void ExponentialBackoffPolicyDoublesWithCap() { var policy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.FromMilliseconds(100), MaxDelay = TimeSpan.FromSeconds(1) }; Assert.That(policy.GetDelay(0).TotalMilliseconds, Is.EqualTo(100)); Assert.That(policy.GetDelay(1).TotalMilliseconds, Is.EqualTo(200)); Assert.That(policy.GetDelay(2).TotalMilliseconds, Is.EqualTo(400)); Assert.That(policy.GetDelay(3).TotalMilliseconds, Is.EqualTo(800)); Assert.That(policy.GetDelay(4).TotalMilliseconds, Is.EqualTo(1000)); // capped Assert.That(policy.GetDelay(10).TotalMilliseconds, Is.EqualTo(1000)); } [Test] public void ExponentialBackoffPolicyReturnsInfiniteWhenExhausted() { var policy = new ExponentialBackoffChannelReconnectPolicy { MaxAttempts = 3 }; Assert.That(policy.GetDelay(2), Is.Not.EqualTo(Timeout.InfiniteTimeSpan)); Assert.That(policy.GetDelay(3), Is.EqualTo(Timeout.InfiniteTimeSpan)); Assert.That(policy.GetDelay(100), Is.EqualTo(Timeout.InfiniteTimeSpan)); } [Test] public async Task GetAsyncReturnsManagedChannelWithMatchingKeyAsync() { (ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel channel = await sut.GetAsync(participant, default) .ConfigureAwait(false); Assert.That(channel, Is.Not.Null); Assert.That(channel.Key, Is.EqualTo(ManagedChannelKey.FromEndpoint(endpoint))); Assert.That(channel.State, Is.EqualTo(ChannelState.Ready)); Assert.That(channel.Manager, Is.SameAs(sut)); channel.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task GetAsyncWithFactoryConstructsParticipantInsideLockAsync() { (ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); IManagedTransportChannel? observedLease = null; TestParticipant? participant = null; int factoryCalls = 0; IManagedTransportChannel channel = await sut.GetAsync( endpoint, lease => { factoryCalls++; observedLease = lease; Assert.That(lease.Key, Is.EqualTo(ManagedChannelKey.FromEndpoint(endpoint))); Assert.That(lease.Manager, Is.SameAs(sut)); var createdParticipant = new TestParticipant("p1", endpoint); participant = createdParticipant; return createdParticipant; }, reverseConnection: null, default) .ConfigureAwait(false); Assert.That(factoryCalls, Is.EqualTo(1)); Assert.That(channel, Is.SameAs(observedLease)); Assert.That(participant, Is.Not.Null); Assert.That(channel.State, Is.EqualTo(ChannelState.Ready)); channel.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task GetAsyncWithFactoryExceptionPropagates() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var expected = new InvalidOperationException("factory failed"); Exception? actual = null; int factoryCalls = 0; try { _ = await sut.GetAsync( endpoint, _ => { factoryCalls++; throw expected; }, reverseConnection: null, default) .ConfigureAwait(false); } catch (Exception ex) { actual = ex; } Assert.That(actual, Is.SameAs(expected)); Assert.That(factoryCalls, Is.EqualTo(1)); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel channel = await sut.GetAsync(participant, default) .ConfigureAwait(false); chMock.Verify(c => c.OpenAsync( It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(2)); channel.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task TwoParticipantsSameEndpointShareUnderlyingChannelAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var p1 = new TestParticipant("p1", endpoint); var p2 = new TestParticipant("p2", endpoint); IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false); IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false); Assert.That(ch1.Key, Is.EqualTo(ch2.Key)); Assert.That(ch1, Is.Not.SameAs(ch2)); // distinct lease wrappers // underlying transport opened exactly once chMock.Verify(c => c.OpenAsync( It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); ch1.Dispose(); ch2.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task ReleasingOneLeaseKeepsChannelAliveForOtherAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var p1 = new TestParticipant("p1", endpoint); var p2 = new TestParticipant("p2", endpoint); IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false); IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false); ch1.Dispose(); // underlying channel should still be alive Assert.That(ch2.State, Is.EqualTo(ChannelState.Ready)); chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Never); ch2.Dispose(); // give the close fiber a moment to run await Task.Delay(100).ConfigureAwait(false); } finally { serverCert.Dispose(); } } [Test] public async Task DiscoveryClientCreateAsyncSharesSessionChannelAndReleasesLeaseAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { var endpointConfiguration = new EndpointConfiguration { OperationTimeout = 6000 }; ConfiguredEndpoint endpoint = GetNoneSecurityEndpoint(endpointConfiguration); var sessionParticipant = new TestParticipant("session", endpoint); IManagedTransportChannel sessionChannel = await sut.GetAsync(sessionParticipant, default) .ConfigureAwait(false); DiscoveryClient? discoveryClient = null; try { discoveryClient = await DiscoveryClient.CreateAsync( sut, new Uri(endpoint.Description.EndpointUrl!), endpointConfiguration, NUnitTelemetryContext.Create(), ct: default).ConfigureAwait(false); Assert.That(discoveryClient.TransportChannel, Is.InstanceOf()); var discoveryChannel = (IManagedTransportChannel)discoveryClient.TransportChannel; Assert.That(discoveryChannel.Key, Is.EqualTo(sessionChannel.Key)); Assert.That(discoveryChannel, Is.Not.SameAs(sessionChannel)); chMock.Verify(c => c.OpenAsync( It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); discoveryClient.Dispose(); discoveryClient = null; chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Never); Assert.That(sessionChannel.State, Is.EqualTo(ChannelState.Ready)); } finally { discoveryClient?.Dispose(); sessionChannel.Dispose(); } // lease.Dispose() is non-blocking; poll for the // CloseAsync invocation before the strict verify. await WaitForMockInvocationAsync( () => chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Once)) .ConfigureAwait(false); } finally { serverCert.Dispose(); } } [Test] public async Task RegistrationClientCreateAsyncUsesManagedLeaseAndReleasesItAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetNoneSecurityEndpoint(new EndpointConfiguration { OperationTimeout = 6000 }); RegistrationClient registrationClient = await RegistrationClient.CreateAsync( sut, endpoint.Description, endpoint.Configuration, NUnitTelemetryContext.Create(), ct: default).ConfigureAwait(false); Assert.That(registrationClient.TransportChannel, Is.InstanceOf()); chMock.Verify(c => c.OpenAsync( It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); registrationClient.Dispose(); chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Once); } finally { serverCert.Dispose(); } } [Test] public async Task ReconnectAsyncNotifiesAttachedParticipantsAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAsync(ch, default).ConfigureAwait(false); Assert.That(participant.NotificationCount, Is.GreaterThanOrEqualTo(1)); Assert.That(ch.State, Is.EqualTo(ChannelState.Ready)); ch.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task ConcurrentReconnectCallsCoalesceIntoOneCycleAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); int reconnectCalls = 0; chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(async () => { Interlocked.Increment(ref reconnectCalls); await Task.Delay(100).ConfigureAwait(false); }); Task t1 = sut.ReconnectAsync(ch, default).AsTask(); Task t2 = sut.ReconnectAsync(ch, default).AsTask(); Task t3 = sut.ReconnectAsync(ch, default).AsTask(); await Task.WhenAll(t1, t2, t3).ConfigureAwait(false); Assert.That(reconnectCalls, Is.EqualTo(1), "Concurrent ReconnectAsync calls should coalesce into a single cycle."); Assert.That(participant.NotificationCount, Is.EqualTo(1)); ch.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task FatalForChannelTransitionsToFaultedStateAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant( "p1", endpoint, onReconnect: (_, _, _) => ParticipantReconnectResult.FatalForChannel); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAsync(ch, default).ConfigureAwait(false); Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted)); ch.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task FatalForParticipantDetachesOnlyThatParticipantAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var p1 = new TestParticipant( "p1", endpoint, onReconnect: (_, _, _) => ParticipantReconnectResult.FatalForParticipant); var p2 = new TestParticipant("p2", endpoint); IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false); IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAsync(ch1, default).ConfigureAwait(false); Assert.That(ch1.State, Is.EqualTo(ChannelState.Ready)); Assert.That(ch2.State, Is.EqualTo(ChannelState.Ready)); // p2 still got the notification Assert.That(p2.NotificationCount, Is.GreaterThanOrEqualTo(1)); ch1.Dispose(); ch2.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task ReconnectAllAsyncTriggersAllEntriesAsync() { (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var p1 = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(p1, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAllAsync(default).ConfigureAwait(false); Assert.That(p1.NotificationCount, Is.GreaterThanOrEqualTo(1)); Assert.That(ch.State, Is.EqualTo(ChannelState.Ready)); ch.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task RebindParticipantSwapsParticipantOnLeaseAsync() { (ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var bootstrap = new TestParticipant("boot", endpoint); IManagedTransportChannel ch = await sut.GetAsync(bootstrap, default) .ConfigureAwait(false); var real = new TestParticipant("real", endpoint); #pragma warning disable CS0618 // Test verifies the obsolete compatibility shim remains functional. sut.RebindParticipant(ch, real); #pragma warning restore CS0618 // No exception means swap succeeded; participant list is internal, // but downstream OnReconnect tests verify behavior. ch.Dispose(); } finally { serverCert.Dispose(); } } [Test] public async Task FailoverWithDifferentEndpointSwapsLeaseAsync() { var harness = new SessionChannelHarness(); ConfiguredEndpoint endpointA = GetSessionEndpoint("opc.tcp://localhost:4840"); ConfiguredEndpoint endpointB = GetSessionEndpoint("opc.tcp://localhost:4841"); Session? session = null; try { session = await harness.CreateSessionAsync(endpointA).ConfigureAwait(false); IManagedTransportChannel oldLease = session.ManagedChannel!; ScriptedChannel oldTransport = harness.CreatedChannels[0]; await session.RecreateInPlaceAsync(endpointB, ct: default).ConfigureAwait(false); IManagedTransportChannel newLease = session.ManagedChannel!; Assert.That(newLease, Is.Not.SameAs(oldLease)); Assert.That(newLease.Key, Is.EqualTo(ManagedChannelKey.FromEndpoint(endpointB))); Assert.That(newLease.State, Is.EqualTo(ChannelState.Ready)); // RecreateInPlaceAsync swaps the lease and fires the // old-lease teardown asynchronously (the underlying // CloseAsync runs on the thread pool). Poll for both // the lease-state transition AND the underlying close // before the hard assertions. await WaitForConditionAsync( () => oldLease.State == ChannelState.Closed && oldTransport.CloseCount == 1, "oldLease.State == Closed && oldTransport.CloseCount == 1").ConfigureAwait(false); Assert.That(oldLease.State, Is.EqualTo(ChannelState.Closed)); Assert.That(oldTransport.CloseCount, Is.EqualTo(1)); Assert.That(harness.CreatedChannels, Has.Count.EqualTo(2)); Assert.That( harness.CreatedChannels[1].OpenedEndpointUrl, Is.EqualTo(endpointB.Description.EndpointUrl)); } finally { session?.Dispose(); await harness.DisposeAsync().ConfigureAwait(false); } } [Test] public async Task FailoverWithSameEndpointKeepsLeaseAsync() { var harness = new SessionChannelHarness(); ConfiguredEndpoint endpoint = GetSessionEndpoint("opc.tcp://localhost:4840"); Session? session = null; try { session = await harness.CreateSessionAsync(endpoint).ConfigureAwait(false); IManagedTransportChannel oldLease = session.ManagedChannel!; ScriptedChannel transport = harness.CreatedChannels[0]; transport.SupportedFeatures = TransportChannelFeatures.Reconnect; await session.RecreateInPlaceAsync(ct: default).ConfigureAwait(false); Assert.That(session.ManagedChannel, Is.SameAs(oldLease)); Assert.That(transport.ReconnectCount, Is.EqualTo(1)); Assert.That(transport.CloseCount, Is.Zero); Assert.That(harness.CreatedChannels, Has.Count.EqualTo(1)); } finally { session?.Dispose(); await harness.DisposeAsync().ConfigureAwait(false); } } [Test] public async Task FailoverWithExplicitChannelTakesLegacyPathAsync() { var harness = new SessionChannelHarness(); ConfiguredEndpoint endpointA = GetSessionEndpoint("opc.tcp://localhost:4840"); ConfiguredEndpoint endpointB = GetSessionEndpoint("opc.tcp://localhost:4841"); Session? session = null; try { session = await harness.CreateSessionAsync(endpointA).ConfigureAwait(false); int managerCreatedChannels = harness.CreatedChannels.Count; ScriptedChannel explicitChannel = harness.CreateOpenedStandaloneChannel(endpointB); await session .RecreateInPlaceAsync(endpointB, channel: explicitChannel.Channel, ct: default) .ConfigureAwait(false); Assert.That(harness.CreatedChannels, Has.Count.EqualTo(managerCreatedChannels)); Assert.That(session.TransportChannel, Is.SameAs(explicitChannel.Channel)); Assert.That(explicitChannel.SendRequestCount, Is.GreaterThan(0)); Assert.That(explicitChannel.CloseCount, Is.Zero); } finally { session?.Dispose(); await harness.DisposeAsync().ConfigureAwait(false); } } [Test] [NonParallelizable] public async Task MetricsAreEmittedForChannelLifetimeAsync() { using var metrics = new ChannelMetricListener(); ITelemetryContext telemetry = NUnitTelemetryContext.Create(); (ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(telemetry); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); string endpointUrl = endpoint.Description.EndpointUrl!; var p1 = new TestParticipant("p1", endpoint); var p2 = new TestParticipant("p2", endpoint); IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false); IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false); metrics.RecordObservableInstruments(); ch1.Dispose(); ch2.Dispose(); // ch.Dispose() is non-blocking (lease teardown runs on // the threadpool). The teardown emits the close counter // *before* the active up/down counter, so waiting on the // close measurement alone still races the active(-1) one // that the assertions below require: poll for the last // measurement the teardown produces instead. await WaitForMeasurementAsync( metrics, "opc.ua.channel.active", -1, Tag("endpoint", endpointUrl)).ConfigureAwait(false); Assert.That(metrics.HasMeasurement( "opc.ua.channel.open", Tag("endpoint", endpointUrl), Tag("reverse", false)), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.active", 1, Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.active", -1, Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.refcount", 2, Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.participants", 2, Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.close", Tag("endpoint", endpointUrl), Tag("reverse", false), Tag("reason", "lease-released")), Is.True, metrics.FormatMeasurements()); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] [NonParallelizable] public async Task MetricsAreEmittedForReconnectAndGateWaitAsync() { using var metrics = new ChannelMetricListener(); ITelemetryContext telemetry = NUnitTelemetryContext.Create(); (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(telemetry); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); string endpointUrl = endpoint.Description.EndpointUrl!; var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); var reconnectEntered = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); var allowReconnect = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(() => new ValueTask(WaitForReconnectReleaseAsync( reconnectEntered, allowReconnect))); chMock.Setup(c => c.SendRequestAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask(new ReadResponse { ResponseHeader = new ResponseHeader { ServiceResult = StatusCodes.Good } })); Task reconnectTask = sut.ReconnectAsync(ch, default).AsTask(); await reconnectEntered.Task.WaitAsync(s_completionTimeout).ConfigureAwait(false); Task sendTask = ch.SendRequestAsync( new ReadRequest { RequestHeader = new RequestHeader() }, default).AsTask(); allowReconnect.SetResult(true); await Task.WhenAll(reconnectTask, sendTask).ConfigureAwait(false); // RunReconnectCycleAsync.RecordReconnectDuration is emitted from a // finally block AFTER tcs.TrySetResult(true) returns, so on a fast // runner the test thread can race past the assertion before the // measurement lands. Poll briefly for the histogram measurement. await WaitForMeasurementAsync( metrics, "opc.ua.channel.reconnect.duration", Tag("endpoint", endpointUrl), Tag("outcome", "success")) .ConfigureAwait(false); Assert.That(metrics.HasMeasurement( "opc.ua.channel.reconnect.attempts", Tag("endpoint", endpointUrl), Tag("outcome", "success")), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.reconnect.duration", Tag("endpoint", endpointUrl), Tag("outcome", "success")), Is.True, metrics.FormatMeasurements()); Assert.That(metrics.HasMeasurement( "opc.ua.channel.gate.wait", Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements()); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task SendRequestTransparentlyRetriesIdempotentRequestOnTransientDropAsync() { var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.Zero, MaxDelay = TimeSpan.Zero }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); var sendCount = 0; chMock.Setup(c => c.SendRequestAsync( It.IsAny(), It.IsAny())) .Returns((_, _) => { if (Interlocked.Increment(ref sendCount) == 1) { throw ServiceResultException.Create( StatusCodes.BadConnectionClosed, "simulated in-flight drop"); } return new ValueTask(new ReadResponse { ResponseHeader = new ResponseHeader { ServiceResult = StatusCodes.Good } }); }); IServiceResponse response = await ch.SendRequestAsync( new ReadRequest { RequestHeader = new RequestHeader() }, default).ConfigureAwait(false); Assert.That(response, Is.Not.Null); Assert.That( sendCount, Is.EqualTo(2), "The idempotent request should be transparently resent after the channel recovered."); chMock.Verify(c => c.ReconnectAsync( It.IsAny(), It.IsAny()), Times.AtLeastOnce); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task SendRequestDoesNotRetryNonIdempotentRequestOnTransientDropAsync() { var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.Zero, MaxDelay = TimeSpan.Zero }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); var sendCount = 0; chMock.Setup(c => c.SendRequestAsync( It.IsAny(), It.IsAny())) .Returns((_, _) => { Interlocked.Increment(ref sendCount); throw ServiceResultException.Create( StatusCodes.BadConnectionClosed, "simulated in-flight drop"); }); ServiceResultException ex = await AssertThrowsAsync( ch.SendRequestAsync( new WriteRequest { RequestHeader = new RequestHeader() }, default).AsTask(), TimeSpan.FromSeconds(30)).ConfigureAwait(false); Assert.That(ex, Is.Not.Null); Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadConnectionClosed)); Assert.That( sendCount, Is.EqualTo(1), "A non-idempotent request must not be resent; the error surfaces for higher-level recovery."); chMock.Verify(c => c.ReconnectAsync( It.IsAny(), It.IsAny()), Times.Never); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task SendRequestDoesNotRetryNonTransientErrorAsync() { var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.Zero, MaxDelay = TimeSpan.Zero }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); var sendCount = 0; chMock.Setup(c => c.SendRequestAsync( It.IsAny(), It.IsAny())) .Returns((_, _) => { Interlocked.Increment(ref sendCount); throw ServiceResultException.Create( StatusCodes.BadNodeIdUnknown, "non-transient application error"); }); ServiceResultException ex = await AssertThrowsAsync( ch.SendRequestAsync( new ReadRequest { RequestHeader = new RequestHeader() }, default).AsTask(), TimeSpan.FromSeconds(30)).ConfigureAwait(false); Assert.That(ex, Is.Not.Null); Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadNodeIdUnknown)); Assert.That( sendCount, Is.EqualTo(1), "A non-transient error must surface immediately without a reconnect."); chMock.Verify(c => c.ReconnectAsync( It.IsAny(), It.IsAny()), Times.Never); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task ReconnectGenerationIncrementsOnTransportReconnectingAsync() { var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.Zero, MaxDelay = TimeSpan.Zero }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); object entry = GetLeaseEntry(ch); long before = (long)GetInternalPropertyValue(entry, "ReconnectGeneration"); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAsync(ch, default).ConfigureAwait(false); long after = (long)GetInternalPropertyValue(entry, "ReconnectGeneration"); Assert.That( after, Is.GreaterThan(before), "Entering TransportReconnecting must bump the reconnect generation."); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } private static Task WaitForMeasurementAsync( ChannelMetricListener metrics, string instrumentName, params KeyValuePair[] tags) { return WaitForMeasurementAsync(metrics, instrumentName, null, tags); } private static async Task WaitForMeasurementAsync( ChannelMetricListener metrics, string instrumentName, double? value, params KeyValuePair[] tags) { const int kMaxPollMs = 2000; const int kPollIntervalMs = 25; int elapsed = 0; while (!metrics.HasMeasurement(instrumentName, value, tags) && elapsed < kMaxPollMs) { await Task.Delay(kPollIntervalMs).ConfigureAwait(false); elapsed += kPollIntervalMs; } } private static async Task WaitForMockInvocationAsync(Action verify) { // Used to bridge Moq.Verify against state mutated by fire- // and-forget tasks (e.g. ManagedTransportChannelLease.Dispose // which posts the actual underlying CloseAsync onto the // thread pool). Polls the verify until it stops throwing or // the budget is exhausted; the final invocation is allowed // to throw and surface the failure to NUnit. const int kMaxPollMs = 2000; const int kPollIntervalMs = 25; int elapsed = 0; while (elapsed < kMaxPollMs) { try { verify(); return; } catch (MockException) { await Task.Delay(kPollIntervalMs).ConfigureAwait(false); elapsed += kPollIntervalMs; } } verify(); } private static async Task WaitForConditionAsync( Func condition, string description) { // Generic test-side poll for state mutated by fire-and-forget // tasks. Returns once the condition holds; if the budget is // exhausted the caller's subsequent assertion is allowed to // run and surface the failure to NUnit. const int kMaxPollMs = 2000; const int kPollIntervalMs = 25; int elapsed = 0; while (!condition() && elapsed < kMaxPollMs) { await Task.Delay(kPollIntervalMs).ConfigureAwait(false); elapsed += kPollIntervalMs; } _ = description; } [Test] [NonParallelizable] public async Task ActivitySpanIsRecordedForReconnectAsync() { using var listener = new ChannelActivityListener(); ITelemetryContext telemetry = NUnitTelemetryContext.Create(); (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(telemetry); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); string endpointUrl = endpoint.Description.EndpointUrl!; var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAsync(ch, default).ConfigureAwait(false); Activity activity = await listener .WaitForStoppedActivityAsync("OpcUaChannelReconnect") .ConfigureAwait(false); var tags = activity.TagObjects.ToDictionary(t => t.Key, t => t.Value); Assert.That(tags, Does.ContainKey("endpoint")); Assert.That(tags["endpoint"], Is.EqualTo(endpointUrl)); Assert.That(tags, Does.ContainKey("attempt.count")); Assert.That(tags["attempt.count"], Is.EqualTo(1)); Assert.That(tags, Does.ContainKey("outcome")); Assert.That(tags["outcome"], Is.EqualTo("success")); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] [NonParallelizable] public async Task StructuredLogsCaptureStateTransitionsAsync() { using var loggerProvider = new RecordingLoggerProvider(); ITelemetryContext telemetry = DefaultTelemetry.Create( builder => builder.AddProvider(loggerProvider)); (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(telemetry); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect); chMock.Setup(c => c.ReconnectAsync( It.IsAny(), It.IsAny())) .Returns(new ValueTask()); await sut.ReconnectAsync(ch, default).ConfigureAwait(false); ch.Dispose(); // ch.Dispose() is non-blocking; teardown logs are emitted by // the background cleanup task. await WaitForConditionAsync( () => loggerProvider.Records.Any(record => record.EventId.Name == "ParticipantDetached") && loggerProvider.Records.Any(record => record.EventId.Name == "ChannelClosed"), "ParticipantDetached + ChannelClosed events").ConfigureAwait(false); RecordedLogRecord[] records = loggerProvider.Records .Where(record => record.CategoryName == "Opc.Ua.ChannelManager") .ToArray(); string formatted = string.Join( Environment.NewLine, records.Select(record => $"{record.EventId.Name} {record.Message}")); string?[] eventNames = records.Select(record => record.EventId.Name).ToArray(); Assert.That(eventNames, Does.Contain("StateChanged"), formatted); Assert.That(eventNames, Does.Contain("ReconnectStarted"), formatted); Assert.That(eventNames, Does.Contain("ReconnectCompleted"), formatted); Assert.That(eventNames, Does.Contain("ParticipantAttached"), formatted); Assert.That(eventNames, Does.Contain("ParticipantDetached"), formatted); Assert.That(eventNames, Does.Contain("ChannelOpened"), formatted); Assert.That(eventNames, Does.Contain("ChannelClosed"), formatted); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task GetChannelDiagnosticsReturnsSnapshot() { (ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(); try { ConfiguredEndpoint endpoint1 = GetTestEndpoint(serverCert); ConfiguredEndpoint endpoint2 = GetTestEndpoint(serverCert); endpoint2.Description.EndpointUrl = "opc.tcp://localhost:4841"; var p1 = new TestParticipant("p1", endpoint1); var p2 = new TestParticipant("p2", endpoint2); IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false); IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false); IReadOnlyList snapshot = sut.GetChannelDiagnostics(); Assert.That(snapshot, Has.Count.EqualTo(2)); ManagedChannelDiagnostic d1 = snapshot.Single(d => d.Key == ch1.Key); ManagedChannelDiagnostic d2 = snapshot.Single(d => d.Key == ch2.Key); AssertChannelDiagnostic(d1, ch1.Key); AssertChannelDiagnostic(d2, ch2.Key); ch1.Dispose(); ch2.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task ReconnectAsyncWithBudgetStopsWhenExhaustedAsync() { var timeProvider = new FakeTimeProvider(); var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.Zero, MaxDelay = TimeSpan.Zero }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); var budget = new RetryBudget(TimeSpan.Zero, timeProvider); // Await rather than Assert.ThrowsAsync: see AssertThrowsAsync. Task exhaustedReconnect = sut.ReconnectAsync(ch, budget, default).AsTask(); ServiceResultException ex = await AssertThrowsAsync( exhaustedReconnect, s_completionTimeout).ConfigureAwait(false); Assert.That(ex, Is.Not.Null); Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed)); Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted)); chMock.Verify(c => c.ReconnectAsync( It.IsAny(), It.IsAny()), Times.Never); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task ReconnectAsyncSwapsFaultedLeaseEntryAsync() { var timeProvider = new FakeTimeProvider(); var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.FromMilliseconds(100), MaxDelay = TimeSpan.FromMilliseconds(100), MaxAttempts = 3 }; (ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); object originalEntry = GetLeaseEntry(ch); var exhaustedBudget = new RetryBudget(TimeSpan.Zero, timeProvider); // Await rather than Assert.ThrowsAsync: see AssertThrowsAsync. // This test drives a fake clock from this thread, so blocking // here would prevent the clock from ever moving again. Task faultedReconnect = sut .ReconnectAsync(ch, exhaustedBudget, default) .AsTask(); await AssertThrowsAsync( faultedReconnect, s_completionTimeout).ConfigureAwait(false); Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted)); Task reconnectTask = sut.ReconnectAsync(ch, default).AsTask(); await Task.Delay(10).ConfigureAwait(false); Assert.That(reconnectTask.IsCompleted, Is.False, "Swap back-off should delay the reset."); for (int i = 0; i < 4 && !reconnectTask.IsCompleted; i++) { timeProvider.Advance(TimeSpan.FromMilliseconds(100)); await Task.Delay(10).ConfigureAwait(false); } await reconnectTask.WaitAsync(s_completionTimeout).ConfigureAwait(false); object freshEntry = GetLeaseEntry(ch); ManagedChannelDiagnostic diagnostic = sut.GetChannelDiagnostics() .Single(d => d.Key.Equals(ch.Key)); Assert.That(freshEntry, Is.Not.SameAs(originalEntry)); Assert.That(GetEntryState(freshEntry), Is.EqualTo(ChannelState.Ready)); Assert.That(GetInternalIntProperty(freshEntry, "RefCount"), Is.EqualTo(1)); Assert.That(GetInternalIntProperty(freshEntry, "ParticipantCount"), Is.EqualTo(1)); Assert.That(GetInternalIntProperty(originalEntry, "RefCount"), Is.Zero); Assert.That(GetInternalIntProperty(ch, "SwapCount"), Is.EqualTo(1)); Assert.That(diagnostic.Refcount, Is.EqualTo(1)); Assert.That(diagnostic.ParticipantCount, Is.EqualTo(1)); Assert.That(participant.NotificationCount, Is.GreaterThanOrEqualTo(2)); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task ReconnectAsyncWithBudgetShrinksDelayToFitRemainingAsync() { var timeProvider = new ObservableFakeTimeProvider(); var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.FromSeconds(10), MaxDelay = TimeSpan.FromSeconds(10) }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); var budget = new RetryBudget(TimeSpan.FromMilliseconds(100), timeProvider); var reconnecting = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); ch.StateChanged += (_, change) => { if (change.NewState == ChannelState.TransportReconnecting) { reconnecting.TrySetResult(true); } }; // Arm before starting the reconnect so the waiter cannot be // satisfied by an unrelated timer created earlier in the run. Task shrunkBackoff = timeProvider.WaitForTimersCreatedAsync(); Task reconnectTask = sut.ReconnectAsync(ch, budget, default).AsTask(); await reconnecting.Task.WaitAsync(s_completionTimeout).ConfigureAwait(false); await shrunkBackoff.WaitAsync(s_completionTimeout).ConfigureAwait(false); Assert.That(reconnectTask.IsCompleted, Is.False); timeProvider.Advance(TimeSpan.FromMilliseconds(100)); ServiceResultException ex = await AssertThrowsAsync( reconnectTask, s_completionTimeout).ConfigureAwait(false); Assert.That(ex, Is.Not.Null); Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed)); Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted)); chMock.Verify(c => c.ReconnectAsync( It.IsAny(), It.IsAny()), Times.Never); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } [Test] public async Task ReconnectAsyncWithExhaustedPolicyDoesNotSwapTheEntryAsync() { var timeProvider = new ObservableFakeTimeProvider(); var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy { MinDelay = TimeSpan.FromMilliseconds(10), MaxDelay = TimeSpan.FromMilliseconds(10), MaxAttempts = 0 }; (ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider); try { ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert); var participant = new TestParticipant("p1", endpoint); IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false); // Deliberately generous. The policy, not the budget, is what ends this // cycle: GetDelay returns the infinite sentinel as soon as the attempt // count reaches MaxAttempts, before the budget is ever consulted. A // race check that only asked whether the budget still had room would // see plenty here, mistake the deliberate stop for a lost race against // a concurrent close, and swap the entry to run a second, unbudgeted // reconnect cycle behind the swap back-off. var budget = new RetryBudget(TimeSpan.FromMinutes(1), timeProvider); ServiceResultException ex = await AssertThrowsAsync( sut.ReconnectAsync(ch, budget, default).AsTask(), s_completionTimeout).ConfigureAwait(false); Assert.That(ex.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed)); Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted)); Assert.That(GetInternalIntProperty(ch, "SwapCount"), Is.Zero); chMock.Verify(c => c.ReconnectAsync( It.IsAny(), It.IsAny()), Times.Never); ch.Dispose(); } finally { await sut.DisposeAsync().ConfigureAwait(false); serverCert.Dispose(); } } /// /// Awaits and returns the exception it faulted /// with, failing the test if it succeeded or threw something else. /// /// /// Deliberately not Assert.ThrowsAsync. That blocks the calling /// thread until the task completes - sync over async - so on a /// constrained CI agent it can starve the very continuation it is /// waiting for, and it has no timeout, so the block is unbounded. Tests /// that drive a fake clock from the test thread cannot survive either: /// the clock cannot advance while the thread is blocked, NUnit's runner /// thread never returns, and the whole test host hangs until the blame /// collector kills it. /// private static async Task AssertThrowsAsync( Task task, TimeSpan timeout) where TException : Exception { try { await task.WaitAsync(timeout).ConfigureAwait(false); } catch (TException expected) { return expected; } catch (Exception other) { Assert.Fail( $"Expected {typeof(TException).Name} but got " + $"{other.GetType().Name}: {other}"); throw; } Assert.Fail( $"Expected {typeof(TException).Name} but the operation completed successfully."); throw new InvalidOperationException("unreachable"); } private static (ClientChannelManager sut, Certificate serverCert, Mock chMock) CreateMockedSut( ITelemetryContext? telemetry = null, IChannelReconnectPolicy? reconnectPolicy = null, TimeProvider? timeProvider = null) { telemetry ??= NUnitTelemetryContext.Create(); Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA(); var chMock = new Mock(); chMock.Setup(c => c.OpenAsync( It.IsAny(), It.IsAny(), It.IsAny())) .Callback( (_, settings, _) => DisposeMockOwnedCertificates(settings)) .Returns(new ValueTask()); chMock.Setup(c => c.OpenAsync( It.IsAny(), It.IsAny(), It.IsAny())) .Callback( (_, settings, _) => DisposeMockOwnedCertificates(settings)) .Returns(new ValueTask()); chMock.Setup(c => c.CloseAsync(It.IsAny())) .Returns(new ValueTask()); chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.None); var bindings = new Mock(); bindings.Setup(b => b.Create(It.IsAny(), It.IsAny())) .Returns(chMock.Object); var configuration = new ApplicationConfiguration(telemetry); var sut = new ClientChannelManager( configuration, telemetry, bindings.Object, reconnectPolicy, timeProvider); return (sut, serverCert, chMock); } private static void DisposeMockOwnedCertificates(TransportChannelSettings settings) { settings.ServerCertificate?.Dispose(); settings.ServerCertificate = null; } private static ConfiguredEndpoint GetTestEndpoint(Certificate serverCert) { var endpoint = new ConfiguredEndpoint { Configuration = new EndpointConfiguration { OperationTimeout = 6000 } }; endpoint.Description.EndpointUrl = "opc.tcp://localhost:4840"; endpoint.Description.SecurityMode = MessageSecurityMode.SignAndEncrypt; endpoint.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256; endpoint.Description.ServerCertificate = serverCert.RawData.ToByteString(); return endpoint; } private static ConfiguredEndpoint GetNoneSecurityEndpoint(EndpointConfiguration endpointConfiguration) { var description = new EndpointDescription { EndpointUrl = "opc.tcp://localhost:4840", SecurityMode = MessageSecurityMode.None, SecurityPolicyUri = SecurityPolicies.None }; description.Server.ApplicationUri = description.EndpointUrl; description.Server.ApplicationType = ApplicationType.DiscoveryServer; return new ConfiguredEndpoint(null, description, endpointConfiguration) { UpdateBeforeConnect = false }; } private static ConfiguredEndpoint GetSessionEndpoint(string endpointUrl) { return SessionChannelHarness.CreateEndpoint(endpointUrl); } private static KeyValuePair Tag(string key, object? value) { return new KeyValuePair(key, value); } private static async Task WaitForReconnectReleaseAsync( TaskCompletionSource reconnectEntered, TaskCompletionSource allowReconnect) { reconnectEntered.TrySetResult(true); await allowReconnect.Task.ConfigureAwait(false); } private static object GetLeaseEntry(IManagedTransportChannel channel) { return GetInternalPropertyValue(channel, "Entry"); } private static ChannelState GetEntryState(object entry) { return (ChannelState)GetInternalPropertyValue(entry, "State"); } private static int GetInternalIntProperty(object target, string propertyName) { return (int)GetInternalPropertyValue(target, propertyName); } private static object GetInternalPropertyValue(object target, string propertyName) { const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; PropertyInfo? property = target.GetType().GetProperty(propertyName, flags); Assert.That(property, Is.Not.Null, $"Expected property {propertyName} on {target.GetType()}."); object? value = property!.GetValue(target); Assert.That(value, Is.Not.Null, $"Expected property {propertyName} to return a value."); return value!; } private static void AssertChannelDiagnostic( ManagedChannelDiagnostic diagnostic, ManagedChannelKey key) { Assert.That(diagnostic.Key, Is.EqualTo(key)); Assert.That(diagnostic.State, Is.EqualTo(ChannelState.Ready)); Assert.That(diagnostic.Refcount, Is.EqualTo(1)); Assert.That(diagnostic.ParticipantCount, Is.EqualTo(1)); Assert.That(diagnostic.OpenedAt, Is.Not.Default); Assert.That(diagnostic.LastStateChange, Is.Not.Default); Assert.That(diagnostic.LastReconnectAttempt, Is.Zero); Assert.That(diagnostic.LastError, Is.Null); } public interface IChannel : ITransportChannel, ISecureChannel; private sealed class ChannelActivityListener : IDisposable { public ChannelActivityListener() { m_listener = new ActivityListener { ShouldListenTo = source => source.Name == "Opc.Ua.ChannelManager", Sample = (ref _) => ActivitySamplingResult.AllDataAndRecorded, ActivityStopped = activity => { StoppedActivities.Add(activity); _ = m_stoppedActivity.TrySetResult(activity); } }; ActivitySource.AddActivityListener(m_listener); } public List StoppedActivities { get; } = []; public async Task WaitForStoppedActivityAsync(string operationName) { Activity activity = await m_stoppedActivity.Task .WaitAsync(s_completionTimeout) .ConfigureAwait(false); Assert.That(activity.OperationName, Is.EqualTo(operationName)); return activity; } public void Dispose() { m_listener.Dispose(); } private readonly ActivityListener m_listener; private readonly TaskCompletionSource m_stoppedActivity = new( TaskCreationOptions.RunContinuationsAsynchronously); } private sealed class ChannelMetricListener : IDisposable { public ChannelMetricListener() { m_listener = new MeterListener { InstrumentPublished = (instrument, listener) => { if (instrument.Name.StartsWith("opc.ua.channel.", StringComparison.Ordinal)) { listener.EnableMeasurementEvents(instrument); } } }; m_listener.SetMeasurementEventCallback(OnLongMeasurementRecorded); m_listener.SetMeasurementEventCallback(OnDoubleMeasurementRecorded); m_listener.Start(); } public ConcurrentQueue Measurements { get; } = new(); public void RecordObservableInstruments() { m_listener.RecordObservableInstruments(); } public bool HasMeasurement( string instrumentName, params KeyValuePair[] tags) { return HasMeasurement(instrumentName, null, tags); } public bool HasMeasurement( string instrumentName, double? value, params KeyValuePair[] tags) { // Snapshot under enumeration to avoid races with the // metric callbacks that fire concurrently from the // channel manager's threadpool teardown work. foreach (MeasurementRecord m in Measurements) { if (m.InstrumentName == instrumentName && (value == null || m.Value == value.Value) && tags.All(tag => m.Tags.TryGetValue(tag.Key, out object? actual) && Equals(actual, tag.Value))) { return true; } } return false; } public string FormatMeasurements() { var builder = new StringBuilder(); // ConcurrentQueue enumeration is snapshot-stable. foreach (MeasurementRecord measurement in Measurements) { string tags = string.Join( ", ", measurement.Tags.Select(t => $"{t.Key}={t.Value}")); builder .Append(measurement.InstrumentName) .Append('=') .Append(measurement.Value.ToString(CultureInfo.InvariantCulture)) .Append(" {") .Append(tags) .AppendLine("}"); } return builder.ToString(); } public void Dispose() { m_listener.Dispose(); } private void OnLongMeasurementRecorded( Instrument instrument, long measurement, ReadOnlySpan> tags, object? state) { Measurements.Enqueue(new MeasurementRecord(instrument.Name, measurement, tags.ToArray())); } private void OnDoubleMeasurementRecorded( Instrument instrument, double measurement, ReadOnlySpan> tags, object? state) { Measurements.Enqueue(new MeasurementRecord(instrument.Name, measurement, tags.ToArray())); } private readonly MeterListener m_listener; } private sealed class MeasurementRecord { public MeasurementRecord( string instrumentName, double value, KeyValuePair[] tags) { InstrumentName = instrumentName; Value = value; Tags = tags.ToDictionary(t => t.Key, t => t.Value); } public string InstrumentName { get; } public double Value { get; } public Dictionary Tags { get; } } private sealed class TestParticipant : IReconnectParticipant { private readonly Func? m_onReconnect; private int m_notificationCount; public TestParticipant( string id, ConfiguredEndpoint endpoint, Func? onReconnect = null) { Id = id; Endpoint = endpoint; m_onReconnect = onReconnect; } public string Id { get; } public ConfiguredEndpoint Endpoint { get; } public int NotificationCount => Volatile.Read(ref m_notificationCount); /// public IRetryBudget? CreateReconnectBudget(TimeProvider timeProvider) { return null; } public ValueTask OnReconnectAsync( IManagedTransportChannel channel, int reconnectAttempt, CancellationToken ct) { Interlocked.Increment(ref m_notificationCount); ParticipantReconnectResult result = m_onReconnect?.Invoke(channel, reconnectAttempt, ct) ?? ParticipantReconnectResult.Reactivated; return new ValueTask(result); } } /// /// A that lets a test wait until the code /// under test has actually registered its back-off timers. /// /// /// Waiting is deliberately relative: a waiter is armed for "N /// more timers from now" rather than for "the Nth timer of the run". /// Absolute numbering is a race - anything else that happens to create a /// timer on this provider first (an earlier reconnect in the same test, /// or manager housekeeping) consumes the low numbers, the waiter then /// completes before the timer the test cares about exists, and the /// subsequent Advance fires nothing. The reconnect is left parked on a /// fake clock that nobody will move again, which hangs the test - and, /// because NUnit blocks the runner thread, the whole test host. /// private sealed class ObservableFakeTimeProvider : FakeTimeProvider { public override ITimer CreateTimer( TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) { ITimer timer = base.CreateTimer(callback, state, dueTime, period); List>? ready = null; lock (m_lock) { m_timerCount++; for (int i = m_waiters.Count - 1; i >= 0; i--) { if (m_timerCount >= m_waiters[i].Target) { (ready ??= []).Add(m_waiters[i].Completion); m_waiters.RemoveAt(i); } } } if (ready != null) { foreach (TaskCompletionSource completion in ready) { completion.TrySetResult(true); } } return timer; } /// /// Returns a task that completes once /// further timers have been created, counted from this call. Arm it /// before starting the operation whose timers are awaited. /// public Task WaitForTimersCreatedAsync(int count = 1) { if (count < 1) { throw new ArgumentOutOfRangeException(nameof(count)); } var completion = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); lock (m_lock) { m_waiters.Add((m_timerCount + count, completion)); } return completion.Task; } private readonly Lock m_lock = new(); private readonly List<(int Target, TaskCompletionSource Completion)> m_waiters = []; private int m_timerCount; } } }