/* ========================================================================
* Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved.
*
* OPC Foundation MIT License 1.00
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
* The complete license agreement can be found here:
* http://opcfoundation.org/License/MIT/1.00/
* ======================================================================*/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Globalization;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Time.Testing;
using Moq;
using NUnit.Framework;
using Opc.Ua.Bindings;
using Opc.Ua.Client.Tests.Stack.Client.Fakes;
using Opc.Ua.Security.Certificates;
using Opc.Ua.Tests;
// CA2000: test code; many disposables are ownership-transferred to test fixtures or short-lived.
#pragma warning disable CA2000
namespace Opc.Ua.Client.Tests.Stack.Client
{
///
/// Unit tests for the managed (sharing, refcount, coalesced
/// reconnect, participant notification) behavior of
/// implementing
/// .
///
[TestFixture]
[Category("Client")]
[SetCulture("en-us")]
[SetUICulture("en-us")]
[Parallelizable]
public sealed class ClientChannelManagerManagedTests
{
private static readonly ICertificateFactory s_factory = DefaultCertificateFactory.Instance;
///
/// How long a test waits for an asynchronous operation it has already
/// unblocked to be observed as complete.
///
///
/// This is a hang detector, not a latency assertion. These tests drive a
/// fake clock, so the work under test finishes in microseconds; what is
/// being waited on is purely the thread-pool scheduling of the
/// continuation chain. On a saturated CI agent the pool injects threads
/// at roughly one per second, so a short budget times out while the
/// operation is merely queued - the channel has already faulted and its
/// completion source has already been signalled. Keep this generous
/// enough that it never fires on a healthy run, and far below the
/// blame-hang timeout so a genuine deadlock still fails the job quickly.
///
private static readonly TimeSpan s_completionTimeout = TimeSpan.FromSeconds(60);
[Test]
public void ChannelKeyEqualityIsValueBased()
{
using Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA();
ConfiguredEndpoint endpoint1 = GetTestEndpoint(serverCert);
ConfiguredEndpoint endpoint2 = GetTestEndpoint(serverCert);
var k1 = ManagedChannelKey.FromEndpoint(endpoint1);
var k2 = ManagedChannelKey.FromEndpoint(endpoint2);
Assert.That(k1, Is.EqualTo(k2));
Assert.That(k1.GetHashCode(), Is.EqualTo(k2.GetHashCode()));
}
[Test]
public void ChannelKeyDistinguishesReverseFromForward()
{
using Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA();
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var forward = ManagedChannelKey.FromEndpoint(endpoint);
var reverse = ManagedChannelKey.FromEndpoint(
endpoint,
reverseConnectionIdentity: new object());
Assert.That(forward, Is.Not.EqualTo(reverse));
}
[Test]
public void ChannelKeyDistinguishesDifferentReverseHandles()
{
using Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA();
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var r1 = ManagedChannelKey.FromEndpoint(endpoint, reverseConnectionIdentity: new object());
var r2 = ManagedChannelKey.FromEndpoint(endpoint, reverseConnectionIdentity: new object());
Assert.That(r1, Is.Not.EqualTo(r2));
}
[Test]
public void ExponentialBackoffPolicyDoublesWithCap()
{
var policy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.FromMilliseconds(100),
MaxDelay = TimeSpan.FromSeconds(1)
};
Assert.That(policy.GetDelay(0).TotalMilliseconds, Is.EqualTo(100));
Assert.That(policy.GetDelay(1).TotalMilliseconds, Is.EqualTo(200));
Assert.That(policy.GetDelay(2).TotalMilliseconds, Is.EqualTo(400));
Assert.That(policy.GetDelay(3).TotalMilliseconds, Is.EqualTo(800));
Assert.That(policy.GetDelay(4).TotalMilliseconds, Is.EqualTo(1000)); // capped
Assert.That(policy.GetDelay(10).TotalMilliseconds, Is.EqualTo(1000));
}
[Test]
public void ExponentialBackoffPolicyReturnsInfiniteWhenExhausted()
{
var policy = new ExponentialBackoffChannelReconnectPolicy
{
MaxAttempts = 3
};
Assert.That(policy.GetDelay(2), Is.Not.EqualTo(Timeout.InfiniteTimeSpan));
Assert.That(policy.GetDelay(3), Is.EqualTo(Timeout.InfiniteTimeSpan));
Assert.That(policy.GetDelay(100), Is.EqualTo(Timeout.InfiniteTimeSpan));
}
[Test]
public async Task GetAsyncReturnsManagedChannelWithMatchingKeyAsync()
{
(ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel channel = await sut.GetAsync(participant, default)
.ConfigureAwait(false);
Assert.That(channel, Is.Not.Null);
Assert.That(channel.Key, Is.EqualTo(ManagedChannelKey.FromEndpoint(endpoint)));
Assert.That(channel.State, Is.EqualTo(ChannelState.Ready));
Assert.That(channel.Manager, Is.SameAs(sut));
channel.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task GetAsyncWithFactoryConstructsParticipantInsideLockAsync()
{
(ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
IManagedTransportChannel? observedLease = null;
TestParticipant? participant = null;
int factoryCalls = 0;
IManagedTransportChannel channel = await sut.GetAsync(
endpoint,
lease =>
{
factoryCalls++;
observedLease = lease;
Assert.That(lease.Key, Is.EqualTo(ManagedChannelKey.FromEndpoint(endpoint)));
Assert.That(lease.Manager, Is.SameAs(sut));
var createdParticipant = new TestParticipant("p1", endpoint);
participant = createdParticipant;
return createdParticipant;
},
reverseConnection: null,
default)
.ConfigureAwait(false);
Assert.That(factoryCalls, Is.EqualTo(1));
Assert.That(channel, Is.SameAs(observedLease));
Assert.That(participant, Is.Not.Null);
Assert.That(channel.State, Is.EqualTo(ChannelState.Ready));
channel.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task GetAsyncWithFactoryExceptionPropagates()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var expected = new InvalidOperationException("factory failed");
Exception? actual = null;
int factoryCalls = 0;
try
{
_ = await sut.GetAsync(
endpoint,
_ =>
{
factoryCalls++;
throw expected;
},
reverseConnection: null,
default)
.ConfigureAwait(false);
}
catch (Exception ex)
{
actual = ex;
}
Assert.That(actual, Is.SameAs(expected));
Assert.That(factoryCalls, Is.EqualTo(1));
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel channel = await sut.GetAsync(participant, default)
.ConfigureAwait(false);
chMock.Verify(c => c.OpenAsync(
It.IsAny(),
It.IsAny(),
It.IsAny()),
Times.Exactly(2));
channel.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task TwoParticipantsSameEndpointShareUnderlyingChannelAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var p1 = new TestParticipant("p1", endpoint);
var p2 = new TestParticipant("p2", endpoint);
IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false);
IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false);
Assert.That(ch1.Key, Is.EqualTo(ch2.Key));
Assert.That(ch1, Is.Not.SameAs(ch2)); // distinct lease wrappers
// underlying transport opened exactly once
chMock.Verify(c => c.OpenAsync(
It.IsAny(),
It.IsAny(),
It.IsAny()),
Times.Once);
ch1.Dispose();
ch2.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task ReleasingOneLeaseKeepsChannelAliveForOtherAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var p1 = new TestParticipant("p1", endpoint);
var p2 = new TestParticipant("p2", endpoint);
IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false);
IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false);
ch1.Dispose();
// underlying channel should still be alive
Assert.That(ch2.State, Is.EqualTo(ChannelState.Ready));
chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Never);
ch2.Dispose();
// give the close fiber a moment to run
await Task.Delay(100).ConfigureAwait(false);
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task DiscoveryClientCreateAsyncSharesSessionChannelAndReleasesLeaseAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
var endpointConfiguration = new EndpointConfiguration
{
OperationTimeout = 6000
};
ConfiguredEndpoint endpoint = GetNoneSecurityEndpoint(endpointConfiguration);
var sessionParticipant = new TestParticipant("session", endpoint);
IManagedTransportChannel sessionChannel = await sut.GetAsync(sessionParticipant, default)
.ConfigureAwait(false);
DiscoveryClient? discoveryClient = null;
try
{
discoveryClient = await DiscoveryClient.CreateAsync(
sut,
new Uri(endpoint.Description.EndpointUrl!),
endpointConfiguration,
NUnitTelemetryContext.Create(),
ct: default).ConfigureAwait(false);
Assert.That(discoveryClient.TransportChannel, Is.InstanceOf());
var discoveryChannel = (IManagedTransportChannel)discoveryClient.TransportChannel;
Assert.That(discoveryChannel.Key, Is.EqualTo(sessionChannel.Key));
Assert.That(discoveryChannel, Is.Not.SameAs(sessionChannel));
chMock.Verify(c => c.OpenAsync(
It.IsAny(),
It.IsAny(),
It.IsAny()),
Times.Once);
discoveryClient.Dispose();
discoveryClient = null;
chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Never);
Assert.That(sessionChannel.State, Is.EqualTo(ChannelState.Ready));
}
finally
{
discoveryClient?.Dispose();
sessionChannel.Dispose();
}
// lease.Dispose() is non-blocking; poll for the
// CloseAsync invocation before the strict verify.
await WaitForMockInvocationAsync(
() => chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Once))
.ConfigureAwait(false);
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task RegistrationClientCreateAsyncUsesManagedLeaseAndReleasesItAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetNoneSecurityEndpoint(new EndpointConfiguration
{
OperationTimeout = 6000
});
RegistrationClient registrationClient = await RegistrationClient.CreateAsync(
sut,
endpoint.Description,
endpoint.Configuration,
NUnitTelemetryContext.Create(),
ct: default).ConfigureAwait(false);
Assert.That(registrationClient.TransportChannel, Is.InstanceOf());
chMock.Verify(c => c.OpenAsync(
It.IsAny(),
It.IsAny(),
It.IsAny()),
Times.Once);
registrationClient.Dispose();
chMock.Verify(c => c.CloseAsync(It.IsAny()), Times.Once);
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectAsyncNotifiesAttachedParticipantsAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAsync(ch, default).ConfigureAwait(false);
Assert.That(participant.NotificationCount, Is.GreaterThanOrEqualTo(1));
Assert.That(ch.State, Is.EqualTo(ChannelState.Ready));
ch.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task ConcurrentReconnectCallsCoalesceIntoOneCycleAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
int reconnectCalls = 0;
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(async () =>
{
Interlocked.Increment(ref reconnectCalls);
await Task.Delay(100).ConfigureAwait(false);
});
Task t1 = sut.ReconnectAsync(ch, default).AsTask();
Task t2 = sut.ReconnectAsync(ch, default).AsTask();
Task t3 = sut.ReconnectAsync(ch, default).AsTask();
await Task.WhenAll(t1, t2, t3).ConfigureAwait(false);
Assert.That(reconnectCalls, Is.EqualTo(1),
"Concurrent ReconnectAsync calls should coalesce into a single cycle.");
Assert.That(participant.NotificationCount, Is.EqualTo(1));
ch.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task FatalForChannelTransitionsToFaultedStateAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant(
"p1", endpoint,
onReconnect: (_, _, _) => ParticipantReconnectResult.FatalForChannel);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAsync(ch, default).ConfigureAwait(false);
Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted));
ch.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task FatalForParticipantDetachesOnlyThatParticipantAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var p1 = new TestParticipant(
"p1", endpoint,
onReconnect: (_, _, _) => ParticipantReconnectResult.FatalForParticipant);
var p2 = new TestParticipant("p2", endpoint);
IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false);
IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAsync(ch1, default).ConfigureAwait(false);
Assert.That(ch1.State, Is.EqualTo(ChannelState.Ready));
Assert.That(ch2.State, Is.EqualTo(ChannelState.Ready));
// p2 still got the notification
Assert.That(p2.NotificationCount, Is.GreaterThanOrEqualTo(1));
ch1.Dispose();
ch2.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectAllAsyncTriggersAllEntriesAsync()
{
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var p1 = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(p1, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAllAsync(default).ConfigureAwait(false);
Assert.That(p1.NotificationCount, Is.GreaterThanOrEqualTo(1));
Assert.That(ch.State, Is.EqualTo(ChannelState.Ready));
ch.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task RebindParticipantSwapsParticipantOnLeaseAsync()
{
(ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var bootstrap = new TestParticipant("boot", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(bootstrap, default)
.ConfigureAwait(false);
var real = new TestParticipant("real", endpoint);
#pragma warning disable CS0618 // Test verifies the obsolete compatibility shim remains functional.
sut.RebindParticipant(ch, real);
#pragma warning restore CS0618
// No exception means swap succeeded; participant list is internal,
// but downstream OnReconnect tests verify behavior.
ch.Dispose();
}
finally
{
serverCert.Dispose();
}
}
[Test]
public async Task FailoverWithDifferentEndpointSwapsLeaseAsync()
{
var harness = new SessionChannelHarness();
ConfiguredEndpoint endpointA = GetSessionEndpoint("opc.tcp://localhost:4840");
ConfiguredEndpoint endpointB = GetSessionEndpoint("opc.tcp://localhost:4841");
Session? session = null;
try
{
session = await harness.CreateSessionAsync(endpointA).ConfigureAwait(false);
IManagedTransportChannel oldLease = session.ManagedChannel!;
ScriptedChannel oldTransport = harness.CreatedChannels[0];
await session.RecreateInPlaceAsync(endpointB, ct: default).ConfigureAwait(false);
IManagedTransportChannel newLease = session.ManagedChannel!;
Assert.That(newLease, Is.Not.SameAs(oldLease));
Assert.That(newLease.Key, Is.EqualTo(ManagedChannelKey.FromEndpoint(endpointB)));
Assert.That(newLease.State, Is.EqualTo(ChannelState.Ready));
// RecreateInPlaceAsync swaps the lease and fires the
// old-lease teardown asynchronously (the underlying
// CloseAsync runs on the thread pool). Poll for both
// the lease-state transition AND the underlying close
// before the hard assertions.
await WaitForConditionAsync(
() => oldLease.State == ChannelState.Closed &&
oldTransport.CloseCount == 1,
"oldLease.State == Closed && oldTransport.CloseCount == 1").ConfigureAwait(false);
Assert.That(oldLease.State, Is.EqualTo(ChannelState.Closed));
Assert.That(oldTransport.CloseCount, Is.EqualTo(1));
Assert.That(harness.CreatedChannels, Has.Count.EqualTo(2));
Assert.That(
harness.CreatedChannels[1].OpenedEndpointUrl,
Is.EqualTo(endpointB.Description.EndpointUrl));
}
finally
{
session?.Dispose();
await harness.DisposeAsync().ConfigureAwait(false);
}
}
[Test]
public async Task FailoverWithSameEndpointKeepsLeaseAsync()
{
var harness = new SessionChannelHarness();
ConfiguredEndpoint endpoint = GetSessionEndpoint("opc.tcp://localhost:4840");
Session? session = null;
try
{
session = await harness.CreateSessionAsync(endpoint).ConfigureAwait(false);
IManagedTransportChannel oldLease = session.ManagedChannel!;
ScriptedChannel transport = harness.CreatedChannels[0];
transport.SupportedFeatures = TransportChannelFeatures.Reconnect;
await session.RecreateInPlaceAsync(ct: default).ConfigureAwait(false);
Assert.That(session.ManagedChannel, Is.SameAs(oldLease));
Assert.That(transport.ReconnectCount, Is.EqualTo(1));
Assert.That(transport.CloseCount, Is.Zero);
Assert.That(harness.CreatedChannels, Has.Count.EqualTo(1));
}
finally
{
session?.Dispose();
await harness.DisposeAsync().ConfigureAwait(false);
}
}
[Test]
public async Task FailoverWithExplicitChannelTakesLegacyPathAsync()
{
var harness = new SessionChannelHarness();
ConfiguredEndpoint endpointA = GetSessionEndpoint("opc.tcp://localhost:4840");
ConfiguredEndpoint endpointB = GetSessionEndpoint("opc.tcp://localhost:4841");
Session? session = null;
try
{
session = await harness.CreateSessionAsync(endpointA).ConfigureAwait(false);
int managerCreatedChannels = harness.CreatedChannels.Count;
ScriptedChannel explicitChannel = harness.CreateOpenedStandaloneChannel(endpointB);
await session
.RecreateInPlaceAsync(endpointB, channel: explicitChannel.Channel, ct: default)
.ConfigureAwait(false);
Assert.That(harness.CreatedChannels, Has.Count.EqualTo(managerCreatedChannels));
Assert.That(session.TransportChannel, Is.SameAs(explicitChannel.Channel));
Assert.That(explicitChannel.SendRequestCount, Is.GreaterThan(0));
Assert.That(explicitChannel.CloseCount, Is.Zero);
}
finally
{
session?.Dispose();
await harness.DisposeAsync().ConfigureAwait(false);
}
}
[Test]
[NonParallelizable]
public async Task MetricsAreEmittedForChannelLifetimeAsync()
{
using var metrics = new ChannelMetricListener();
ITelemetryContext telemetry = NUnitTelemetryContext.Create();
(ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut(telemetry);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
string endpointUrl = endpoint.Description.EndpointUrl!;
var p1 = new TestParticipant("p1", endpoint);
var p2 = new TestParticipant("p2", endpoint);
IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false);
IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false);
metrics.RecordObservableInstruments();
ch1.Dispose();
ch2.Dispose();
// ch.Dispose() is non-blocking (lease teardown runs on
// the threadpool). The teardown emits the close counter
// *before* the active up/down counter, so waiting on the
// close measurement alone still races the active(-1) one
// that the assertions below require: poll for the last
// measurement the teardown produces instead.
await WaitForMeasurementAsync(
metrics,
"opc.ua.channel.active",
-1,
Tag("endpoint", endpointUrl)).ConfigureAwait(false);
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.open",
Tag("endpoint", endpointUrl),
Tag("reverse", false)), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.active",
1,
Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.active",
-1,
Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.refcount",
2,
Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.participants",
2,
Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.close",
Tag("endpoint", endpointUrl),
Tag("reverse", false),
Tag("reason", "lease-released")), Is.True, metrics.FormatMeasurements());
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
[NonParallelizable]
public async Task MetricsAreEmittedForReconnectAndGateWaitAsync()
{
using var metrics = new ChannelMetricListener();
ITelemetryContext telemetry = NUnitTelemetryContext.Create();
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(telemetry);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
string endpointUrl = endpoint.Description.EndpointUrl!;
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
var reconnectEntered = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
var allowReconnect = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(() => new ValueTask(WaitForReconnectReleaseAsync(
reconnectEntered,
allowReconnect)));
chMock.Setup(c => c.SendRequestAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask(new ReadResponse
{
ResponseHeader = new ResponseHeader { ServiceResult = StatusCodes.Good }
}));
Task reconnectTask = sut.ReconnectAsync(ch, default).AsTask();
await reconnectEntered.Task.WaitAsync(s_completionTimeout).ConfigureAwait(false);
Task sendTask = ch.SendRequestAsync(
new ReadRequest { RequestHeader = new RequestHeader() },
default).AsTask();
allowReconnect.SetResult(true);
await Task.WhenAll(reconnectTask, sendTask).ConfigureAwait(false);
// RunReconnectCycleAsync.RecordReconnectDuration is emitted from a
// finally block AFTER tcs.TrySetResult(true) returns, so on a fast
// runner the test thread can race past the assertion before the
// measurement lands. Poll briefly for the histogram measurement.
await WaitForMeasurementAsync(
metrics,
"opc.ua.channel.reconnect.duration",
Tag("endpoint", endpointUrl),
Tag("outcome", "success"))
.ConfigureAwait(false);
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.reconnect.attempts",
Tag("endpoint", endpointUrl),
Tag("outcome", "success")), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.reconnect.duration",
Tag("endpoint", endpointUrl),
Tag("outcome", "success")), Is.True, metrics.FormatMeasurements());
Assert.That(metrics.HasMeasurement(
"opc.ua.channel.gate.wait",
Tag("endpoint", endpointUrl)), Is.True, metrics.FormatMeasurements());
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task SendRequestTransparentlyRetriesIdempotentRequestOnTransientDropAsync()
{
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.Zero,
MaxDelay = TimeSpan.Zero
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
var sendCount = 0;
chMock.Setup(c => c.SendRequestAsync(
It.IsAny(),
It.IsAny()))
.Returns((_, _) =>
{
if (Interlocked.Increment(ref sendCount) == 1)
{
throw ServiceResultException.Create(
StatusCodes.BadConnectionClosed, "simulated in-flight drop");
}
return new ValueTask(new ReadResponse
{
ResponseHeader = new ResponseHeader { ServiceResult = StatusCodes.Good }
});
});
IServiceResponse response = await ch.SendRequestAsync(
new ReadRequest { RequestHeader = new RequestHeader() },
default).ConfigureAwait(false);
Assert.That(response, Is.Not.Null);
Assert.That(
sendCount,
Is.EqualTo(2),
"The idempotent request should be transparently resent after the channel recovered.");
chMock.Verify(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()), Times.AtLeastOnce);
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task SendRequestDoesNotRetryNonIdempotentRequestOnTransientDropAsync()
{
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.Zero,
MaxDelay = TimeSpan.Zero
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
var sendCount = 0;
chMock.Setup(c => c.SendRequestAsync(
It.IsAny(),
It.IsAny()))
.Returns((_, _) =>
{
Interlocked.Increment(ref sendCount);
throw ServiceResultException.Create(
StatusCodes.BadConnectionClosed, "simulated in-flight drop");
});
ServiceResultException ex = await AssertThrowsAsync(
ch.SendRequestAsync(
new WriteRequest { RequestHeader = new RequestHeader() },
default).AsTask(),
TimeSpan.FromSeconds(30)).ConfigureAwait(false);
Assert.That(ex, Is.Not.Null);
Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadConnectionClosed));
Assert.That(
sendCount,
Is.EqualTo(1),
"A non-idempotent request must not be resent; the error surfaces for higher-level recovery.");
chMock.Verify(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()), Times.Never);
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task SendRequestDoesNotRetryNonTransientErrorAsync()
{
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.Zero,
MaxDelay = TimeSpan.Zero
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
var sendCount = 0;
chMock.Setup(c => c.SendRequestAsync(
It.IsAny(),
It.IsAny()))
.Returns((_, _) =>
{
Interlocked.Increment(ref sendCount);
throw ServiceResultException.Create(
StatusCodes.BadNodeIdUnknown, "non-transient application error");
});
ServiceResultException ex = await AssertThrowsAsync(
ch.SendRequestAsync(
new ReadRequest { RequestHeader = new RequestHeader() },
default).AsTask(),
TimeSpan.FromSeconds(30)).ConfigureAwait(false);
Assert.That(ex, Is.Not.Null);
Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadNodeIdUnknown));
Assert.That(
sendCount,
Is.EqualTo(1),
"A non-transient error must surface immediately without a reconnect.");
chMock.Verify(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()), Times.Never);
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectGenerationIncrementsOnTransportReconnectingAsync()
{
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.Zero,
MaxDelay = TimeSpan.Zero
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
object entry = GetLeaseEntry(ch);
long before = (long)GetInternalPropertyValue(entry, "ReconnectGeneration");
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAsync(ch, default).ConfigureAwait(false);
long after = (long)GetInternalPropertyValue(entry, "ReconnectGeneration");
Assert.That(
after,
Is.GreaterThan(before),
"Entering TransportReconnecting must bump the reconnect generation.");
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
private static Task WaitForMeasurementAsync(
ChannelMetricListener metrics,
string instrumentName,
params KeyValuePair[] tags)
{
return WaitForMeasurementAsync(metrics, instrumentName, null, tags);
}
private static async Task WaitForMeasurementAsync(
ChannelMetricListener metrics,
string instrumentName,
double? value,
params KeyValuePair[] tags)
{
const int kMaxPollMs = 2000;
const int kPollIntervalMs = 25;
int elapsed = 0;
while (!metrics.HasMeasurement(instrumentName, value, tags) && elapsed < kMaxPollMs)
{
await Task.Delay(kPollIntervalMs).ConfigureAwait(false);
elapsed += kPollIntervalMs;
}
}
private static async Task WaitForMockInvocationAsync(Action verify)
{
// Used to bridge Moq.Verify against state mutated by fire-
// and-forget tasks (e.g. ManagedTransportChannelLease.Dispose
// which posts the actual underlying CloseAsync onto the
// thread pool). Polls the verify until it stops throwing or
// the budget is exhausted; the final invocation is allowed
// to throw and surface the failure to NUnit.
const int kMaxPollMs = 2000;
const int kPollIntervalMs = 25;
int elapsed = 0;
while (elapsed < kMaxPollMs)
{
try
{
verify();
return;
}
catch (MockException)
{
await Task.Delay(kPollIntervalMs).ConfigureAwait(false);
elapsed += kPollIntervalMs;
}
}
verify();
}
private static async Task WaitForConditionAsync(
Func condition,
string description)
{
// Generic test-side poll for state mutated by fire-and-forget
// tasks. Returns once the condition holds; if the budget is
// exhausted the caller's subsequent assertion is allowed to
// run and surface the failure to NUnit.
const int kMaxPollMs = 2000;
const int kPollIntervalMs = 25;
int elapsed = 0;
while (!condition() && elapsed < kMaxPollMs)
{
await Task.Delay(kPollIntervalMs).ConfigureAwait(false);
elapsed += kPollIntervalMs;
}
_ = description;
}
[Test]
[NonParallelizable]
public async Task ActivitySpanIsRecordedForReconnectAsync()
{
using var listener = new ChannelActivityListener();
ITelemetryContext telemetry = NUnitTelemetryContext.Create();
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(telemetry);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
string endpointUrl = endpoint.Description.EndpointUrl!;
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAsync(ch, default).ConfigureAwait(false);
Activity activity = await listener
.WaitForStoppedActivityAsync("OpcUaChannelReconnect")
.ConfigureAwait(false);
var tags = activity.TagObjects.ToDictionary(t => t.Key, t => t.Value);
Assert.That(tags, Does.ContainKey("endpoint"));
Assert.That(tags["endpoint"], Is.EqualTo(endpointUrl));
Assert.That(tags, Does.ContainKey("attempt.count"));
Assert.That(tags["attempt.count"], Is.EqualTo(1));
Assert.That(tags, Does.ContainKey("outcome"));
Assert.That(tags["outcome"], Is.EqualTo("success"));
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
[NonParallelizable]
public async Task StructuredLogsCaptureStateTransitionsAsync()
{
using var loggerProvider = new RecordingLoggerProvider();
ITelemetryContext telemetry = DefaultTelemetry.Create(
builder => builder.AddProvider(loggerProvider));
(ClientChannelManager sut, Certificate serverCert, Mock chMock) = CreateMockedSut(telemetry);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.Reconnect);
chMock.Setup(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()))
.Returns(new ValueTask());
await sut.ReconnectAsync(ch, default).ConfigureAwait(false);
ch.Dispose();
// ch.Dispose() is non-blocking; teardown logs are emitted by
// the background cleanup task.
await WaitForConditionAsync(
() => loggerProvider.Records.Any(record =>
record.EventId.Name == "ParticipantDetached") &&
loggerProvider.Records.Any(record =>
record.EventId.Name == "ChannelClosed"),
"ParticipantDetached + ChannelClosed events").ConfigureAwait(false);
RecordedLogRecord[] records = loggerProvider.Records
.Where(record => record.CategoryName == "Opc.Ua.ChannelManager")
.ToArray();
string formatted = string.Join(
Environment.NewLine,
records.Select(record => $"{record.EventId.Name} {record.Message}"));
string?[] eventNames = records.Select(record => record.EventId.Name).ToArray();
Assert.That(eventNames, Does.Contain("StateChanged"), formatted);
Assert.That(eventNames, Does.Contain("ReconnectStarted"), formatted);
Assert.That(eventNames, Does.Contain("ReconnectCompleted"), formatted);
Assert.That(eventNames, Does.Contain("ParticipantAttached"), formatted);
Assert.That(eventNames, Does.Contain("ParticipantDetached"), formatted);
Assert.That(eventNames, Does.Contain("ChannelOpened"), formatted);
Assert.That(eventNames, Does.Contain("ChannelClosed"), formatted);
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task GetChannelDiagnosticsReturnsSnapshot()
{
(ClientChannelManager sut, Certificate serverCert, _) = CreateMockedSut();
try
{
ConfiguredEndpoint endpoint1 = GetTestEndpoint(serverCert);
ConfiguredEndpoint endpoint2 = GetTestEndpoint(serverCert);
endpoint2.Description.EndpointUrl = "opc.tcp://localhost:4841";
var p1 = new TestParticipant("p1", endpoint1);
var p2 = new TestParticipant("p2", endpoint2);
IManagedTransportChannel ch1 = await sut.GetAsync(p1, default).ConfigureAwait(false);
IManagedTransportChannel ch2 = await sut.GetAsync(p2, default).ConfigureAwait(false);
IReadOnlyList snapshot = sut.GetChannelDiagnostics();
Assert.That(snapshot, Has.Count.EqualTo(2));
ManagedChannelDiagnostic d1 = snapshot.Single(d => d.Key == ch1.Key);
ManagedChannelDiagnostic d2 = snapshot.Single(d => d.Key == ch2.Key);
AssertChannelDiagnostic(d1, ch1.Key);
AssertChannelDiagnostic(d2, ch2.Key);
ch1.Dispose();
ch2.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectAsyncWithBudgetStopsWhenExhaustedAsync()
{
var timeProvider = new FakeTimeProvider();
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.Zero,
MaxDelay = TimeSpan.Zero
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
var budget = new RetryBudget(TimeSpan.Zero, timeProvider);
// Await rather than Assert.ThrowsAsync: see AssertThrowsAsync.
Task exhaustedReconnect = sut.ReconnectAsync(ch, budget, default).AsTask();
ServiceResultException ex = await AssertThrowsAsync(
exhaustedReconnect,
s_completionTimeout).ConfigureAwait(false);
Assert.That(ex, Is.Not.Null);
Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed));
Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted));
chMock.Verify(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()),
Times.Never);
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectAsyncSwapsFaultedLeaseEntryAsync()
{
var timeProvider = new FakeTimeProvider();
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.FromMilliseconds(100),
MaxDelay = TimeSpan.FromMilliseconds(100),
MaxAttempts = 3
};
(ClientChannelManager sut, Certificate serverCert, _) =
CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
object originalEntry = GetLeaseEntry(ch);
var exhaustedBudget = new RetryBudget(TimeSpan.Zero, timeProvider);
// Await rather than Assert.ThrowsAsync: see AssertThrowsAsync.
// This test drives a fake clock from this thread, so blocking
// here would prevent the clock from ever moving again.
Task faultedReconnect = sut
.ReconnectAsync(ch, exhaustedBudget, default)
.AsTask();
await AssertThrowsAsync(
faultedReconnect,
s_completionTimeout).ConfigureAwait(false);
Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted));
Task reconnectTask = sut.ReconnectAsync(ch, default).AsTask();
await Task.Delay(10).ConfigureAwait(false);
Assert.That(reconnectTask.IsCompleted, Is.False, "Swap back-off should delay the reset.");
for (int i = 0; i < 4 && !reconnectTask.IsCompleted; i++)
{
timeProvider.Advance(TimeSpan.FromMilliseconds(100));
await Task.Delay(10).ConfigureAwait(false);
}
await reconnectTask.WaitAsync(s_completionTimeout).ConfigureAwait(false);
object freshEntry = GetLeaseEntry(ch);
ManagedChannelDiagnostic diagnostic = sut.GetChannelDiagnostics()
.Single(d => d.Key.Equals(ch.Key));
Assert.That(freshEntry, Is.Not.SameAs(originalEntry));
Assert.That(GetEntryState(freshEntry), Is.EqualTo(ChannelState.Ready));
Assert.That(GetInternalIntProperty(freshEntry, "RefCount"), Is.EqualTo(1));
Assert.That(GetInternalIntProperty(freshEntry, "ParticipantCount"), Is.EqualTo(1));
Assert.That(GetInternalIntProperty(originalEntry, "RefCount"), Is.Zero);
Assert.That(GetInternalIntProperty(ch, "SwapCount"), Is.EqualTo(1));
Assert.That(diagnostic.Refcount, Is.EqualTo(1));
Assert.That(diagnostic.ParticipantCount, Is.EqualTo(1));
Assert.That(participant.NotificationCount, Is.GreaterThanOrEqualTo(2));
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectAsyncWithBudgetShrinksDelayToFitRemainingAsync()
{
var timeProvider = new ObservableFakeTimeProvider();
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.FromSeconds(10),
MaxDelay = TimeSpan.FromSeconds(10)
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
var budget = new RetryBudget(TimeSpan.FromMilliseconds(100), timeProvider);
var reconnecting = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
ch.StateChanged += (_, change) =>
{
if (change.NewState == ChannelState.TransportReconnecting)
{
reconnecting.TrySetResult(true);
}
};
// Arm before starting the reconnect so the waiter cannot be
// satisfied by an unrelated timer created earlier in the run.
Task shrunkBackoff = timeProvider.WaitForTimersCreatedAsync();
Task reconnectTask = sut.ReconnectAsync(ch, budget, default).AsTask();
await reconnecting.Task.WaitAsync(s_completionTimeout).ConfigureAwait(false);
await shrunkBackoff.WaitAsync(s_completionTimeout).ConfigureAwait(false);
Assert.That(reconnectTask.IsCompleted, Is.False);
timeProvider.Advance(TimeSpan.FromMilliseconds(100));
ServiceResultException ex = await AssertThrowsAsync(
reconnectTask,
s_completionTimeout).ConfigureAwait(false);
Assert.That(ex, Is.Not.Null);
Assert.That(ex!.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed));
Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted));
chMock.Verify(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()),
Times.Never);
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
[Test]
public async Task ReconnectAsyncWithExhaustedPolicyDoesNotSwapTheEntryAsync()
{
var timeProvider = new ObservableFakeTimeProvider();
var reconnectPolicy = new ExponentialBackoffChannelReconnectPolicy
{
MinDelay = TimeSpan.FromMilliseconds(10),
MaxDelay = TimeSpan.FromMilliseconds(10),
MaxAttempts = 0
};
(ClientChannelManager sut, Certificate serverCert, Mock chMock) =
CreateMockedSut(reconnectPolicy: reconnectPolicy, timeProvider: timeProvider);
try
{
ConfiguredEndpoint endpoint = GetTestEndpoint(serverCert);
var participant = new TestParticipant("p1", endpoint);
IManagedTransportChannel ch = await sut.GetAsync(participant, default).ConfigureAwait(false);
// Deliberately generous. The policy, not the budget, is what ends this
// cycle: GetDelay returns the infinite sentinel as soon as the attempt
// count reaches MaxAttempts, before the budget is ever consulted. A
// race check that only asked whether the budget still had room would
// see plenty here, mistake the deliberate stop for a lost race against
// a concurrent close, and swap the entry to run a second, unbudgeted
// reconnect cycle behind the swap back-off.
var budget = new RetryBudget(TimeSpan.FromMinutes(1), timeProvider);
ServiceResultException ex = await AssertThrowsAsync(
sut.ReconnectAsync(ch, budget, default).AsTask(),
s_completionTimeout).ConfigureAwait(false);
Assert.That(ex.StatusCode, Is.EqualTo(StatusCodes.BadSecureChannelClosed));
Assert.That(ch.State, Is.EqualTo(ChannelState.Faulted));
Assert.That(GetInternalIntProperty(ch, "SwapCount"), Is.Zero);
chMock.Verify(c => c.ReconnectAsync(
It.IsAny(),
It.IsAny()),
Times.Never);
ch.Dispose();
}
finally
{
await sut.DisposeAsync().ConfigureAwait(false);
serverCert.Dispose();
}
}
///
/// Awaits and returns the exception it faulted
/// with, failing the test if it succeeded or threw something else.
///
///
/// Deliberately not Assert.ThrowsAsync. That blocks the calling
/// thread until the task completes - sync over async - so on a
/// constrained CI agent it can starve the very continuation it is
/// waiting for, and it has no timeout, so the block is unbounded. Tests
/// that drive a fake clock from the test thread cannot survive either:
/// the clock cannot advance while the thread is blocked, NUnit's runner
/// thread never returns, and the whole test host hangs until the blame
/// collector kills it.
///
private static async Task AssertThrowsAsync(
Task task,
TimeSpan timeout)
where TException : Exception
{
try
{
await task.WaitAsync(timeout).ConfigureAwait(false);
}
catch (TException expected)
{
return expected;
}
catch (Exception other)
{
Assert.Fail(
$"Expected {typeof(TException).Name} but got " +
$"{other.GetType().Name}: {other}");
throw;
}
Assert.Fail(
$"Expected {typeof(TException).Name} but the operation completed successfully.");
throw new InvalidOperationException("unreachable");
}
private static (ClientChannelManager sut, Certificate serverCert, Mock chMock) CreateMockedSut(
ITelemetryContext? telemetry = null,
IChannelReconnectPolicy? reconnectPolicy = null,
TimeProvider? timeProvider = null)
{
telemetry ??= NUnitTelemetryContext.Create();
Certificate serverCert = s_factory.CreateCertificate("CN=server").CreateForRSA();
var chMock = new Mock();
chMock.Setup(c => c.OpenAsync(
It.IsAny(),
It.IsAny(),
It.IsAny()))
.Callback(
(_, settings, _) => DisposeMockOwnedCertificates(settings))
.Returns(new ValueTask());
chMock.Setup(c => c.OpenAsync(
It.IsAny(),
It.IsAny(),
It.IsAny()))
.Callback(
(_, settings, _) => DisposeMockOwnedCertificates(settings))
.Returns(new ValueTask());
chMock.Setup(c => c.CloseAsync(It.IsAny()))
.Returns(new ValueTask());
chMock.Setup(c => c.SupportedFeatures).Returns(TransportChannelFeatures.None);
var bindings = new Mock();
bindings.Setup(b => b.Create(It.IsAny(), It.IsAny()))
.Returns(chMock.Object);
var configuration = new ApplicationConfiguration(telemetry);
var sut = new ClientChannelManager(
configuration,
telemetry,
bindings.Object,
reconnectPolicy,
timeProvider);
return (sut, serverCert, chMock);
}
private static void DisposeMockOwnedCertificates(TransportChannelSettings settings)
{
settings.ServerCertificate?.Dispose();
settings.ServerCertificate = null;
}
private static ConfiguredEndpoint GetTestEndpoint(Certificate serverCert)
{
var endpoint = new ConfiguredEndpoint
{
Configuration = new EndpointConfiguration
{
OperationTimeout = 6000
}
};
endpoint.Description.EndpointUrl = "opc.tcp://localhost:4840";
endpoint.Description.SecurityMode = MessageSecurityMode.SignAndEncrypt;
endpoint.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256;
endpoint.Description.ServerCertificate = serverCert.RawData.ToByteString();
return endpoint;
}
private static ConfiguredEndpoint GetNoneSecurityEndpoint(EndpointConfiguration endpointConfiguration)
{
var description = new EndpointDescription
{
EndpointUrl = "opc.tcp://localhost:4840",
SecurityMode = MessageSecurityMode.None,
SecurityPolicyUri = SecurityPolicies.None
};
description.Server.ApplicationUri = description.EndpointUrl;
description.Server.ApplicationType = ApplicationType.DiscoveryServer;
return new ConfiguredEndpoint(null, description, endpointConfiguration)
{
UpdateBeforeConnect = false
};
}
private static ConfiguredEndpoint GetSessionEndpoint(string endpointUrl)
{
return SessionChannelHarness.CreateEndpoint(endpointUrl);
}
private static KeyValuePair Tag(string key, object? value)
{
return new KeyValuePair(key, value);
}
private static async Task WaitForReconnectReleaseAsync(
TaskCompletionSource reconnectEntered,
TaskCompletionSource allowReconnect)
{
reconnectEntered.TrySetResult(true);
await allowReconnect.Task.ConfigureAwait(false);
}
private static object GetLeaseEntry(IManagedTransportChannel channel)
{
return GetInternalPropertyValue(channel, "Entry");
}
private static ChannelState GetEntryState(object entry)
{
return (ChannelState)GetInternalPropertyValue(entry, "State");
}
private static int GetInternalIntProperty(object target, string propertyName)
{
return (int)GetInternalPropertyValue(target, propertyName);
}
private static object GetInternalPropertyValue(object target, string propertyName)
{
const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
PropertyInfo? property = target.GetType().GetProperty(propertyName, flags);
Assert.That(property, Is.Not.Null, $"Expected property {propertyName} on {target.GetType()}.");
object? value = property!.GetValue(target);
Assert.That(value, Is.Not.Null, $"Expected property {propertyName} to return a value.");
return value!;
}
private static void AssertChannelDiagnostic(
ManagedChannelDiagnostic diagnostic,
ManagedChannelKey key)
{
Assert.That(diagnostic.Key, Is.EqualTo(key));
Assert.That(diagnostic.State, Is.EqualTo(ChannelState.Ready));
Assert.That(diagnostic.Refcount, Is.EqualTo(1));
Assert.That(diagnostic.ParticipantCount, Is.EqualTo(1));
Assert.That(diagnostic.OpenedAt, Is.Not.Default);
Assert.That(diagnostic.LastStateChange, Is.Not.Default);
Assert.That(diagnostic.LastReconnectAttempt, Is.Zero);
Assert.That(diagnostic.LastError, Is.Null);
}
public interface IChannel : ITransportChannel, ISecureChannel;
private sealed class ChannelActivityListener : IDisposable
{
public ChannelActivityListener()
{
m_listener = new ActivityListener
{
ShouldListenTo = source => source.Name == "Opc.Ua.ChannelManager",
Sample = (ref _) =>
ActivitySamplingResult.AllDataAndRecorded,
ActivityStopped = activity =>
{
StoppedActivities.Add(activity);
_ = m_stoppedActivity.TrySetResult(activity);
}
};
ActivitySource.AddActivityListener(m_listener);
}
public List StoppedActivities { get; } = [];
public async Task WaitForStoppedActivityAsync(string operationName)
{
Activity activity = await m_stoppedActivity.Task
.WaitAsync(s_completionTimeout)
.ConfigureAwait(false);
Assert.That(activity.OperationName, Is.EqualTo(operationName));
return activity;
}
public void Dispose()
{
m_listener.Dispose();
}
private readonly ActivityListener m_listener;
private readonly TaskCompletionSource m_stoppedActivity = new(
TaskCreationOptions.RunContinuationsAsynchronously);
}
private sealed class ChannelMetricListener : IDisposable
{
public ChannelMetricListener()
{
m_listener = new MeterListener
{
InstrumentPublished = (instrument, listener) =>
{
if (instrument.Name.StartsWith("opc.ua.channel.", StringComparison.Ordinal))
{
listener.EnableMeasurementEvents(instrument);
}
}
};
m_listener.SetMeasurementEventCallback(OnLongMeasurementRecorded);
m_listener.SetMeasurementEventCallback(OnDoubleMeasurementRecorded);
m_listener.Start();
}
public ConcurrentQueue Measurements { get; } = new();
public void RecordObservableInstruments()
{
m_listener.RecordObservableInstruments();
}
public bool HasMeasurement(
string instrumentName,
params KeyValuePair[] tags)
{
return HasMeasurement(instrumentName, null, tags);
}
public bool HasMeasurement(
string instrumentName,
double? value,
params KeyValuePair[] tags)
{
// Snapshot under enumeration to avoid races with the
// metric callbacks that fire concurrently from the
// channel manager's threadpool teardown work.
foreach (MeasurementRecord m in Measurements)
{
if (m.InstrumentName == instrumentName &&
(value == null || m.Value == value.Value) &&
tags.All(tag => m.Tags.TryGetValue(tag.Key, out object? actual) &&
Equals(actual, tag.Value)))
{
return true;
}
}
return false;
}
public string FormatMeasurements()
{
var builder = new StringBuilder();
// ConcurrentQueue enumeration is snapshot-stable.
foreach (MeasurementRecord measurement in Measurements)
{
string tags = string.Join(
", ",
measurement.Tags.Select(t => $"{t.Key}={t.Value}"));
builder
.Append(measurement.InstrumentName)
.Append('=')
.Append(measurement.Value.ToString(CultureInfo.InvariantCulture))
.Append(" {")
.Append(tags)
.AppendLine("}");
}
return builder.ToString();
}
public void Dispose()
{
m_listener.Dispose();
}
private void OnLongMeasurementRecorded(
Instrument instrument,
long measurement,
ReadOnlySpan> tags,
object? state)
{
Measurements.Enqueue(new MeasurementRecord(instrument.Name, measurement, tags.ToArray()));
}
private void OnDoubleMeasurementRecorded(
Instrument instrument,
double measurement,
ReadOnlySpan> tags,
object? state)
{
Measurements.Enqueue(new MeasurementRecord(instrument.Name, measurement, tags.ToArray()));
}
private readonly MeterListener m_listener;
}
private sealed class MeasurementRecord
{
public MeasurementRecord(
string instrumentName,
double value,
KeyValuePair[] tags)
{
InstrumentName = instrumentName;
Value = value;
Tags = tags.ToDictionary(t => t.Key, t => t.Value);
}
public string InstrumentName { get; }
public double Value { get; }
public Dictionary Tags { get; }
}
private sealed class TestParticipant : IReconnectParticipant
{
private readonly Func? m_onReconnect;
private int m_notificationCount;
public TestParticipant(
string id,
ConfiguredEndpoint endpoint,
Func? onReconnect = null)
{
Id = id;
Endpoint = endpoint;
m_onReconnect = onReconnect;
}
public string Id { get; }
public ConfiguredEndpoint Endpoint { get; }
public int NotificationCount => Volatile.Read(ref m_notificationCount);
///
public IRetryBudget? CreateReconnectBudget(TimeProvider timeProvider)
{
return null;
}
public ValueTask OnReconnectAsync(
IManagedTransportChannel channel,
int reconnectAttempt,
CancellationToken ct)
{
Interlocked.Increment(ref m_notificationCount);
ParticipantReconnectResult result = m_onReconnect?.Invoke(channel, reconnectAttempt, ct)
?? ParticipantReconnectResult.Reactivated;
return new ValueTask(result);
}
}
///
/// A that lets a test wait until the code
/// under test has actually registered its back-off timers.
///
///
/// Waiting is deliberately relative: a waiter is armed for "N
/// more timers from now" rather than for "the Nth timer of the run".
/// Absolute numbering is a race - anything else that happens to create a
/// timer on this provider first (an earlier reconnect in the same test,
/// or manager housekeeping) consumes the low numbers, the waiter then
/// completes before the timer the test cares about exists, and the
/// subsequent Advance fires nothing. The reconnect is left parked on a
/// fake clock that nobody will move again, which hangs the test - and,
/// because NUnit blocks the runner thread, the whole test host.
///
private sealed class ObservableFakeTimeProvider : FakeTimeProvider
{
public override ITimer CreateTimer(
TimerCallback callback,
object? state,
TimeSpan dueTime,
TimeSpan period)
{
ITimer timer = base.CreateTimer(callback, state, dueTime, period);
List>? ready = null;
lock (m_lock)
{
m_timerCount++;
for (int i = m_waiters.Count - 1; i >= 0; i--)
{
if (m_timerCount >= m_waiters[i].Target)
{
(ready ??= []).Add(m_waiters[i].Completion);
m_waiters.RemoveAt(i);
}
}
}
if (ready != null)
{
foreach (TaskCompletionSource completion in ready)
{
completion.TrySetResult(true);
}
}
return timer;
}
///
/// Returns a task that completes once
/// further timers have been created, counted from this call. Arm it
/// before starting the operation whose timers are awaited.
///
public Task WaitForTimersCreatedAsync(int count = 1)
{
if (count < 1)
{
throw new ArgumentOutOfRangeException(nameof(count));
}
var completion = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
lock (m_lock)
{
m_waiters.Add((m_timerCount + count, completion));
}
return completion.Task;
}
private readonly Lock m_lock = new();
private readonly List<(int Target, TaskCompletionSource Completion)> m_waiters = [];
private int m_timerCount;
}
}
}