/* ========================================================================
* Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved.
*
* OPC Foundation MIT License 1.00
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
* The complete license agreement can be found here:
* http://opcfoundation.org/License/MIT/1.00/
* ======================================================================*/
using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Time.Testing;
using Moq;
using NUnit.Framework;
namespace Opc.Ua.Client.Tests.ManagedSession
{
///
/// Unit tests for .
///
[TestFixture]
[Category("Client")]
[Category("ServerRedundancy")]
public sealed class ServerRedundancyHandlerTests
{
private static readonly DateTime s_now = new(2026, 6, 27, 4, 0, 0, DateTimeKind.Utc);
private Mock m_resolver = null!;
private DefaultServerRedundancyHandler m_handler = null!;
///
/// Creates a strict endpoint resolver mock and a redundancy handler with a fixed clock for each scenario.
///
[SetUp]
public void SetUp()
{
m_resolver = new Mock(MockBehavior.Strict);
m_handler = new DefaultServerRedundancyHandler(m_resolver.Object, new FixedTimeProvider(s_now));
}
///
/// Verifies that server redundancy support values map to the corresponding redundancy modes.
///
[TestCase(0, RedundancySupport.None)]
[TestCase(1, RedundancySupport.Cold)]
[TestCase(2, RedundancySupport.Warm)]
[TestCase(3, RedundancySupport.Hot)]
[TestCase(4, RedundancySupport.Transparent)]
[TestCase(5, RedundancySupport.HotAndMirrored)]
public async Task FetchRedundancyInfoMapsRedundancySupportAsync(
int value,
RedundancySupport expected)
{
Mock mockSession = CreateMockSession(
redundancySupport: value,
serviceLevel: ServiceLevels.HealthyMinimum);
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
Assert.That(info.Mode, Is.EqualTo(expected));
}
[TestCase(RedundancySupport.Cold)]
[TestCase(RedundancySupport.Warm)]
public async Task BasicRedundancyReadRetainsUnresolvedPeersWithoutDiscoveryAsync(RedundancySupport mode)
{
Mock session = CreateMockSession(
(int)mode, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
ServerRedundancyInfo snapshot = await ((IServerRedundancyEndpointCache)m_handler)
.ReadRedundancyInfoAsync(session.Object, CancellationToken.None).ConfigureAwait(false);
Assert.That(snapshot.Mode, Is.EqualTo(mode));
Assert.That(snapshot.ServiceLevel, Is.EqualTo(ServiceLevels.Maximum));
Assert.That(snapshot.RedundantServers.Count, Is.EqualTo(1));
Assert.That(snapshot.RedundantServers[0].ServerUri, Is.EqualTo("urn:offline"));
Assert.That(snapshot.RedundantServers[0].Endpoint, Is.Null);
m_resolver.VerifyNoOtherCalls();
VerifyBatchedRedundancyRead(session);
}
[TestCase(false)]
[TestCase(true)]
public async Task PeerTimeoutPreservesOtherEndpointsAndCoalescesPendingDiscoveryAsync(bool ignoreCancellation)
{
var time = new FakeTimeProvider();
var handler = new DefaultServerRedundancyHandler(m_resolver.Object, time);
ConfiguredEndpoint available = CreateEndpoint("urn:available", "opc.tcp://available:4840");
var blocked = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
var secondLookup = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
int lookups = 0;
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()))
.Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
{
if (Interlocked.Increment(ref lookups) == 2)
{
secondLookup.TrySetResult(true);
}
return new ValueTask(ignoreCancellation
? blocked.Task
: blocked.Task.WaitAsync(ct));
});
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:available", It.IsAny(), It.IsAny()))
.ReturnsAsync(available);
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline", "urn:available"]);
var cache = (IServerRedundancyEndpointCache)handler;
ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
.ConfigureAwait(false);
try
{
Task resolving = cache.ResolveCachedEndpointsAsync(
basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:available", It.IsAny(), It.IsAny()), Times.Once);
Assert.That(resolving.IsCompleted, Is.False);
time.Advance(TimeSpan.FromSeconds(2));
ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(10))
.ConfigureAwait(false);
Assert.That(resolved.Mode, Is.EqualTo(RedundancySupport.Cold));
Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null);
Assert.That(resolved.RedundantServers[1].Endpoint, Is.SameAs(available));
Assert.That(handler.SelectFailoverTarget(resolved, session.Object.ConfiguredEndpoint),
Is.SameAs(available));
Task retry = cache.ResolveCachedEndpointsAsync(
resolved, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
if (!ignoreCancellation)
{
// The timed-out lookup drains on the thread pool, so the retry may first join it and only
// start its own lookup once it has drained. Advancing the clock before then would expire
// the retry's bound too, so wait for the fresh lookup.
await secondLookup.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
}
time.Advance(TimeSpan.FromSeconds(2));
await retry.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()),
Times.Exactly(ignoreCancellation ? 1 : 2));
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:available", It.IsAny(), It.IsAny()), Times.Once);
}
finally
{
blocked.TrySetResult(null);
}
}
[Test]
public async Task ConfiguredPeerDiscoveryTimeoutReplacesTheDefaultBoundAsync()
{
var time = new FakeTimeProvider();
var handler = new DefaultServerRedundancyHandler(
m_resolver.Object,
time,
new ServerRedundancyOptions { PeerDiscoveryTimeout = TimeSpan.FromSeconds(30) });
var blocked = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()))
.Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
new ValueTask(blocked.Task.WaitAsync(ct)));
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
var cache = (IServerRedundancyEndpointCache)handler;
ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
.ConfigureAwait(false);
try
{
Task resolving = cache.ResolveCachedEndpointsAsync(
basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
time.Advance(TimeSpan.FromSeconds(2));
await Task.Yield();
Assert.That(resolving.IsCompleted, Is.False);
time.Advance(TimeSpan.FromSeconds(28));
ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(10))
.ConfigureAwait(false);
Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null);
}
finally
{
blocked.TrySetResult(null);
}
}
[Test]
public void ServerRedundancyOptionsRejectNonPositiveBounds()
{
Assert.Multiple(() =>
{
Assert.That(
() => new ServerRedundancyOptions { RefreshTimeout = TimeSpan.Zero }.Validate(),
Throws.InstanceOf());
Assert.That(
() => new ServerRedundancyOptions
{
PeerDiscoveryTimeout = TimeSpan.FromSeconds(-5)
}.Validate(),
Throws.InstanceOf());
Assert.That(
() => new ServerRedundancyOptions
{
RefreshTimeout = Timeout.InfiniteTimeSpan,
PeerDiscoveryTimeout = Timeout.InfiniteTimeSpan
}.Validate(),
Throws.Nothing);
Assert.That(new ServerRedundancyOptions().RefreshTimeout,
Is.EqualTo(ServerRedundancyOptions.DefaultTimeout));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task PeerDiscoveryTimeoutCancelsResolverBeforeOwnedTimerFiresAsync(bool ignoreCancellation)
{
var clock = new FakeTimeProvider();
var timeProvider = new Mock { CallBase = true };
timeProvider.Setup(provider => provider.GetUtcNow()).Returns(clock.GetUtcNow);
timeProvider.Setup(provider => provider.GetTimestamp()).Returns(clock.GetTimestamp);
timeProvider.SetupGet(provider => provider.TimestampFrequency).Returns(clock.TimestampFrequency);
int timerCount = 0;
timeProvider.Setup(provider => provider.CreateTimer(
It.IsAny(), It.IsAny(),
It.IsAny(), It.IsAny()))
.Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) =>
{
if (Interlocked.Increment(ref timerCount) == 1)
{
// Deliver the outer wait's deadline before the owned cancellation timer.
dueTime += TimeSpan.FromSeconds(1);
}
return clock.CreateTimer(callback, state, dueTime, period);
});
var timeout = TimeSpan.FromSeconds(1);
var handler = new DefaultServerRedundancyHandler(
m_resolver.Object, timeProvider.Object,
new ServerRedundancyOptions { PeerDiscoveryTimeout = timeout });
var blocked = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
CancellationToken discoveryToken = default;
int resolverCalls = 0;
ConfiguredEndpoint freshEndpoint = CreateEndpoint("urn:offline", "opc.tcp://fresh:4840");
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()))
.Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
{
if (Interlocked.Increment(ref resolverCalls) == 1)
{
discoveryToken = ct;
return new ValueTask(
ignoreCancellation ? blocked.Task : blocked.Task.WaitAsync(ct));
}
return new ValueTask(freshEndpoint);
});
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
var cache = (IServerRedundancyEndpointCache)handler;
ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(
session.Object, CancellationToken.None).ConfigureAwait(false);
try
{
Task resolving = cache.ResolveCachedEndpointsAsync(
basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
Assert.That(timerCount, Is.EqualTo(2));
Assert.That(discoveryToken.CanBeCanceled, Is.True);
clock.Advance(timeout);
ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(5))
.ConfigureAwait(false);
Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null);
Assert.That(discoveryToken.IsCancellationRequested, Is.True);
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()), Times.Once);
blocked.TrySetResult(CreateEndpoint("urn:offline", "opc.tcp://late:4840"));
ServerRedundancyInfo retried = await cache.ResolveCachedEndpointsAsync(
basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask()
.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
Assert.That(retried.RedundantServers[0].Endpoint, Is.SameAs(freshEndpoint));
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()), Times.Exactly(2));
}
finally
{
blocked.TrySetResult(null);
}
}
[Test]
public async Task PeerDiscoveryPropagatesCallerCancellationAsync()
{
using var cancellation = new CancellationTokenSource();
var resolverEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var blocked = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
CancellationToken discoveryToken = default;
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:offline", It.IsAny(), It.IsAny()))
.Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
{
discoveryToken = ct;
resolverEntered.TrySetResult(true);
return new ValueTask(blocked.Task.WaitAsync(ct));
});
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
var cache = (IServerRedundancyEndpointCache)m_handler;
ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, cancellation.Token)
.ConfigureAwait(false);
Task resolving = cache.ResolveCachedEndpointsAsync(
basic, session.Object.ConfiguredEndpoint, cancellation.Token).AsTask();
await resolverEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
cancellation.Cancel();
await Assert.ThatAsync(
() => resolving,
Throws.InstanceOf()).ConfigureAwait(false);
Assert.That(discoveryToken.IsCancellationRequested, Is.True);
Assert.That(basic.RedundantServers[0].Endpoint, Is.Null);
}
[Test]
public async Task CancelledMetadataReadDoesNotStartPeerDiscoveryAsync()
{
using var cancellation = new CancellationTokenSource();
cancellation.Cancel();
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
await Assert.ThatAsync(
async () => await m_handler.FetchRedundancyInfoAsync(session.Object, cancellation.Token)
.ConfigureAwait(false),
Throws.InstanceOf()).ConfigureAwait(false);
session.VerifyNoOtherCalls();
m_resolver.VerifyNoOtherCalls();
}
[Test]
public async Task CachedResolutionDoesNotCrossEndpointSecurityProfilesAsync()
{
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:peer"]);
ConfiguredEndpoint unsecured = CreateEndpoint("urn:peer", "opc.tcp://peer:4840");
ConfiguredEndpoint secured = CreateEndpoint("urn:peer", "opc.tcp://peer:4840");
secured.Description.SecurityMode = MessageSecurityMode.Sign;
secured.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256;
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:peer", It.Is(endpoint =>
endpoint.Description.SecurityMode == MessageSecurityMode.None),
It.IsAny()))
.ReturnsAsync(unsecured);
m_resolver.Setup(resolver => resolver.ResolveAsync(
"urn:peer", It.Is(endpoint =>
endpoint.Description.SecurityMode == MessageSecurityMode.Sign),
It.IsAny()))
.ReturnsAsync(secured);
ServerRedundancyInfo first = await m_handler.FetchRedundancyInfoAsync(session.Object)
.ConfigureAwait(false);
Assert.That(first.RedundantServers[0].Endpoint, Is.SameAs(unsecured));
session.Object.ConfiguredEndpoint.Description.SecurityMode = MessageSecurityMode.Sign;
session.Object.ConfiguredEndpoint.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256;
ServerRedundancyInfo second = await m_handler.FetchRedundancyInfoAsync(session.Object)
.ConfigureAwait(false);
Assert.That(second.RedundantServers[0].Endpoint, Is.SameAs(secured));
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:peer", It.IsAny(), It.IsAny()), Times.Exactly(2));
}
[Test]
public async Task RetriedPeerDiscoveryRejectsLateCancelledResultAsync()
{
var time = new FakeTimeProvider();
var handler = new DefaultServerRedundancyHandler(m_resolver.Object, time);
ConfiguredEndpoint stale = CreateEndpoint("urn:peer", "opc.tcp://peer:4840");
ConfiguredEndpoint current = CreateEndpoint("urn:peer", "opc.tcp://peer:4841");
var blocked = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
m_resolver.SetupSequence(resolver => resolver.ResolveAsync(
"urn:peer", It.IsAny(), It.IsAny()))
.Returns(new ValueTask(blocked.Task))
.ReturnsAsync(current);
Mock session = CreateMockSession(
(int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:peer"]);
var cache = (IServerRedundancyEndpointCache)handler;
ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
.ConfigureAwait(false);
Task first = cache.ResolveCachedEndpointsAsync(
basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
time.Advance(TimeSpan.FromSeconds(2));
ServerRedundancyInfo unresolved = await first.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
Assert.That(unresolved.RedundantServers[0].Endpoint, Is.Null);
Task retry = cache.ResolveCachedEndpointsAsync(
unresolved, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:peer", It.IsAny(), It.IsAny()), Times.Once);
blocked.TrySetResult(stale);
ServerRedundancyInfo resolved = await retry.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
Assert.That(resolved.RedundantServers[0].Endpoint, Is.SameAs(current));
ServerRedundancyInfo cached = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
.ConfigureAwait(false);
Assert.That(cached.RedundantServers[0].Endpoint, Is.SameAs(current));
m_resolver.Verify(resolver => resolver.ResolveAsync(
"urn:peer", It.IsAny(), It.IsAny()), Times.Exactly(2));
}
///
/// Verifies that a healthy server avoids proactive failover while retaining a recovery target.
///
[Test]
public void HealthyServerDoesNotProactivelyFailOverButRetainsRecoveryTarget()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.HealthyMinimum,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Healthy,
RedundantServers =
[
CreateServerInfo("urn:backup", ServiceLevels.Maximum, ServerState.Running)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
Assert.That(target, Is.Not.Null);
Assert.That(
target!.Description.Server.ApplicationUri,
Is.EqualTo("urn:backup"));
}
///
/// Verifies that an interface-typed call (the extension method) applies the
/// same health checks as the concrete handler.
///
[Test]
public void InterfaceShouldFailoverDoesNotFailOverFromHealthyServer()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.HealthyMinimum,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Healthy,
RedundantServers =
[
CreateServerInfo("urn:backup", ServiceLevels.Maximum, ServerState.Running)
]
};
IServerRedundancyHandler handler = m_handler;
ServerFailoverDecision decision = handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
}
///
/// Verifies that the extension fallback for a custom handler does not fail
/// over from a Healthy server even when a target is available.
///
[Test]
public void CustomHandlerShouldFailoverDoesNotFailOverFromHealthyServer()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.Maximum,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Healthy
};
var custom = new Mock();
custom.Setup(h => h.SelectFailoverTarget(It.IsAny(), It.IsAny()))
.Returns(CreateCurrentEndpoint("urn:backup"));
ServerFailoverDecision healthy = custom.Object.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
ServerFailoverDecision degraded = custom.Object.ShouldFailover(
new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.DegradedMaximum,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Degraded
},
CreateCurrentEndpoint("urn:current"));
Assert.That(healthy.IsFailoverWarranted, Is.False);
Assert.That(degraded.IsFailoverWarranted, Is.True);
}
///
/// Verifies that a degraded server fails over to a healthy peer.
///
[Test]
public void ShouldFailoverSwitchesFromDegradedToHealthyPeer()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.DegradedMaximum,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Degraded,
RedundantServers =
[
CreateServerInfo("urn:degraded", 150, ServerState.Running),
CreateServerInfo("urn:healthy", 230, ServerState.Running)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.True);
Assert.That(target, Is.Not.Null);
Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy"));
}
///
/// Verifies that a degraded server does not fail over when no healthy peer is available.
///
[Test]
public void ShouldFailoverDoesNotSwitchFromDegradedWithoutHealthyPeer()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = 100,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Degraded,
RedundantServers =
[
CreateServerInfo("urn:degraded", 150, ServerState.Running)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
}
///
/// Verifies that a server in maintenance fails over to an operational peer.
///
[Test]
public void ShouldFailoverSwitchesFromMaintenanceToOperationalPeer()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.Maintenance,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
EstimatedReturnTime = s_now.AddMinutes(10),
RedundantServers =
[
CreateServerInfo("urn:healthy", 230, ServerState.Running)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.True);
Assert.That(target, Is.Not.Null);
Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy"));
}
///
/// Verifies that failover timing honors a maintenance server's estimated return time.
///
[Test]
public void ShouldFailoverHonorsMaintenanceEstimatedReturnTime()
{
DateTime estimatedReturnTime = s_now.AddMinutes(10);
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.Maintenance,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
EstimatedReturnTime = estimatedReturnTime,
RedundantServers =
[
CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
Assert.That(decision.RetryAfter, Is.EqualTo(estimatedReturnTime));
}
///
/// Verifies that failover uses backoff after the maintenance return estimate has elapsed.
///
[Test]
public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeLapsed()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.Maintenance,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
EstimatedReturnTime = s_now.AddMinutes(-1),
RedundantServers =
[
CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
Assert.That(
decision.RetryAfter,
Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff)));
}
///
/// Verifies that failover uses backoff when no maintenance return estimate is available.
///
[Test]
public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeIsAbsent()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.Maintenance,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
EstimatedReturnTime = DateTime.MinValue,
RedundantServers =
[
CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
Assert.That(
decision.RetryAfter,
Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff)));
}
///
/// Verifies that no failover is requested when all peers are down.
///
[Test]
public void ShouldFailoverReturnsNoFailoverWhenAllPeersAreDown()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.NoData,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.NoData,
RedundantServers =
[
CreateServerInfo("urn:shutdown", ServiceLevels.Maximum, ServerState.Shutdown),
CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended)
]
};
ServerFailoverDecision decision = m_handler.ShouldFailover(
info, CreateCurrentEndpoint("urn:current"));
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(decision.IsFailoverWarranted, Is.False);
Assert.That(target, Is.Null);
}
///
/// Verifies that transparent redundancy does not select a client-side failover target.
///
[Test]
public void SelectFailoverTargetReturnsNullForTransparentMode()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Transparent,
ServiceLevel = ServiceLevels.NoData,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.NoData,
RedundantServers =
[
CreateServerInfo("urn:backup", 230, ServerState.Running)
]
};
ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(result, Is.Null);
}
///
/// Verifies that disabled redundancy does not select a failover target.
///
[Test]
public void SelectFailoverTargetReturnsNullForNoneMode()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.None,
ServiceLevel = ServiceLevels.NoData,
ServiceLevelAccessible = false,
ServiceLevelSubrange = ServiceLevelSubrange.NoData,
RedundantServers =
[
CreateServerInfo("urn:backup", 230, ServerState.Running)
]
};
ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(result, Is.Null);
}
///
/// Verifies that failover target selection excludes the current endpoint.
///
[Test]
public void SelectFailoverTargetSkipsCurrentEndpoint()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.NoData,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.NoData,
RedundantServers =
[
CreateServerInfo("urn:current", ServiceLevels.Maximum, ServerState.Running),
CreateServerInfo("urn:backup", 230, ServerState.Running)
]
};
ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(result, Is.Not.Null);
Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:backup"));
}
///
/// Verifies that failover target selection excludes servers that are not running.
///
[Test]
public void SelectFailoverTargetSkipsNonRunningServers()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.NoData,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.NoData,
RedundantServers =
[
CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended),
CreateServerInfo("urn:shutdown", 240, ServerState.Shutdown),
CreateServerInfo("urn:running", 230, ServerState.Running)
]
};
ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(result, Is.Not.Null);
Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:running"));
}
///
/// Verifies that target selection returns null when no viable server remains.
///
[Test]
public void SelectFailoverTargetReturnsNullWhenNoViableServers()
{
var info = new ServerRedundancyInfo
{
Mode = RedundancySupport.Hot,
ServiceLevel = ServiceLevels.NoData,
ServiceLevelAccessible = true,
ServiceLevelSubrange = ServiceLevelSubrange.NoData,
RedundantServers =
[
CreateServerInfo("urn:current", 230, ServerState.Running),
CreateServerInfo("urn:down", 230, ServerState.Shutdown)
]
};
ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(result, Is.Null);
}
///
/// Verifies that redundancy discovery reads the redundant server array.
///
[Test]
public async Task FetchRedundancyInfoReadsRedundantServerArrayAsync()
{
var serverData = new RedundantServerDataType
{
ServerId = "urn:server1",
ServiceLevel = 230,
ServerState = ServerState.Running
};
ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840");
m_resolver.Setup(r => r.ResolveAsync(
"urn:server1",
It.IsAny(),
It.IsAny()))
.ReturnsAsync(resolvedEndpoint);
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: 100,
redundantServers: [serverData]);
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot));
Assert.That(info.ServiceLevelSubrange, Is.EqualTo(ServiceLevelSubrange.Degraded));
Assert.That(info.RedundantServers, Has.Count.EqualTo(1));
Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server1"));
Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint));
}
///
/// Verifies that redundancy discovery resolves server application URIs to endpoints.
///
[Test]
public async Task FetchRedundancyInfoResolvesServerUriArrayToEndpointsAsync()
{
ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840");
m_resolver.Setup(r => r.ResolveAsync(
"urn:server-uri",
It.IsAny(),
It.IsAny()))
.ReturnsAsync(resolvedEndpoint);
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: ServiceLevels.NoData,
serverUris: ["urn:server-uri"]);
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
Assert.That(info.RedundantServers, Has.Count.EqualTo(1));
Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server-uri"));
Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint));
Assert.That(
info.RedundantServers[0].Endpoint!.Description.EndpointUrl,
Is.EqualTo("opc.tcp://server-uri:4840"));
VerifyBatchedRedundancyRead(mockSession);
}
///
/// Verifies that peers with unresolved server URIs are excluded from failover selection.
///
[Test]
public async Task FetchRedundancyInfoExcludesUnresolvedServerUriFromSelectionAsync()
{
m_resolver.Setup(r => r.ResolveAsync(
"urn:unresolved",
It.IsAny(),
It.IsAny()))
.ReturnsAsync((ConfiguredEndpoint?)null);
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: ServiceLevels.NoData,
serverUris: ["urn:unresolved"]);
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(info.RedundantServers, Has.Count.EqualTo(1));
Assert.That(info.RedundantServers[0].Endpoint, Is.Null);
Assert.That(target, Is.Null);
}
///
/// Verifies that a URI-only peer can be selected when its service level is unknown.
///
[Test]
public async Task FetchRedundancyInfoSelectsServerUriOnlyPeerWithUnknownServiceLevelAsync()
{
ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840");
m_resolver.Setup(r => r.ResolveAsync(
"urn:server-uri",
It.IsAny(),
It.IsAny()))
.ReturnsAsync(resolvedEndpoint);
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: ServiceLevels.NoData,
serverUris: ["urn:server-uri"]);
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(info.RedundantServers[0].ServiceLevel, Is.EqualTo(ServiceLevels.NoData));
Assert.That(target, Is.SameAs(resolvedEndpoint));
}
///
/// Verifies that discovery can fall back to a URI-only peer with unknown service level.
///
[Test]
public async Task FetchRedundancyInfoFallsBackToUnknownServerUriOnlyPeerAsync()
{
var serverData = new RedundantServerDataType
{
ServerId = "urn:nodata",
ServiceLevel = ServiceLevels.NoData,
ServerState = ServerState.Running
};
ConfiguredEndpoint noDataEndpoint = CreateEndpoint("urn:nodata", "opc.tcp://nodata:4840");
ConfiguredEndpoint unknownEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840");
m_resolver.Setup(r => r.ResolveAsync(
"urn:nodata",
It.IsAny(),
It.IsAny()))
.ReturnsAsync(noDataEndpoint);
m_resolver.Setup(r => r.ResolveAsync(
"urn:server-uri",
It.IsAny(),
It.IsAny()))
.ReturnsAsync(unknownEndpoint);
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: ServiceLevels.DegradedMaximum,
redundantServers: [serverData],
serverUris: ["urn:nodata", "urn:server-uri"]);
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
info, CreateCurrentEndpoint("urn:current"));
Assert.That(info.RedundantServers, Has.Count.EqualTo(2));
Assert.That(target, Is.SameAs(unknownEndpoint));
}
///
/// Verifies that redundancy discovery caches resolved peer endpoints.
///
[Test]
public async Task FetchRedundancyInfoCachesResolvedEndpointsAsync()
{
var serverData = new RedundantServerDataType
{
ServerId = "urn:server1",
ServiceLevel = 230,
ServerState = ServerState.Running
};
ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840");
m_resolver.Setup(r => r.ResolveAsync(
"urn:server1",
It.IsAny(),
It.IsAny()))
.ReturnsAsync(resolvedEndpoint);
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: 100,
redundantServers: [serverData]);
await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false);
await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false);
m_resolver.Verify(r => r.ResolveAsync(
"urn:server1",
It.IsAny(),
It.IsAny()), Times.Once);
}
///
/// Verifies that transparent redundancy discovery reads the current server identifier.
///
[Test]
public async Task FetchRedundancyInfoReadsTransparentCurrentServerIdAsync()
{
Mock mockSession = CreateMockSession(
redundancySupport: (int)RedundancySupport.Transparent,
serviceLevel: ServiceLevels.HealthyMinimum,
currentServerId: "server-a");
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
Assert.That(info.CurrentServerId, Is.EqualTo("server-a"));
}
///
/// Verifies that failed redundancy reads report no redundancy, an inaccessible service level, and no peers.
///
[Test]
public async Task FetchRedundancyInfoHandlesReadErrorsAsync()
{
Mock mockSession = CreateMockSessionWithBadStatus();
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
Assert.That(info.Mode, Is.EqualTo(RedundancySupport.None));
Assert.That(info.ServiceLevel, Is.Zero);
Assert.That(info.ServiceLevelAccessible, Is.False);
Assert.That(info.RedundantServers, Is.Empty);
}
///
/// Verifies that malformed optional redundancy nodes preserve the discovered mode without inventing peers.
///
[Test]
public async Task FetchRedundancyInfoHandlesMalformedOptionalNodesAsync()
{
Mock mockSession = CreateMockSessionWithThrowingOptionalReads();
ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
mockSession.Object).ConfigureAwait(false);
Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot));
Assert.That(info.RedundantServers, Is.Empty);
}
///
/// Verifies that discovery and failover APIs reject null sessions, redundancy information, and endpoints.
///
[Test]
public void NullArgumentsThrow()
{
var info = new ServerRedundancyInfo();
ConfiguredEndpoint endpoint = CreateCurrentEndpoint("urn:current");
Assert.That(
async () => await m_handler.FetchRedundancyInfoAsync(null!).ConfigureAwait(false),
Throws.ArgumentNullException);
Assert.That(() => m_handler.ShouldFailover(null!, endpoint), Throws.ArgumentNullException);
Assert.That(() => m_handler.ShouldFailover(info, null!), Throws.ArgumentNullException);
Assert.That(() => m_handler.SelectFailoverTarget(null!, endpoint), Throws.ArgumentNullException);
Assert.That(() => m_handler.SelectFailoverTarget(info, null!), Throws.ArgumentNullException);
}
private static RedundantServer CreateServerInfo(
string uri,
byte serviceLevel,
ServerState state)
{
return new RedundantServer
{
ServerUri = uri,
ServiceLevel = serviceLevel,
ServerState = state,
Endpoint = CreateEndpoint(uri, $"opc.tcp://{uri[4..]}:4840")
};
}
private static ConfiguredEndpoint CreateCurrentEndpoint(string applicationUri)
{
return CreateEndpoint(applicationUri, "opc.tcp://current:4840");
}
private static ConfiguredEndpoint CreateEndpoint(string applicationUri, string endpointUrl)
{
var description = new EndpointDescription
{
EndpointUrl = endpointUrl,
SecurityMode = MessageSecurityMode.None,
SecurityPolicyUri = SecurityPolicies.None,
Server = new ApplicationDescription
{
ApplicationUri = applicationUri,
DiscoveryUrls = new ArrayOf(new[] { endpointUrl })
}
};
return new ConfiguredEndpoint(null, description, configuration: null);
}
private static Mock CreateMockSession(
int redundancySupport,
byte serviceLevel,
RedundantServerDataType[]? redundantServers = null,
string[]? serverUris = null,
string currentServerId = "")
{
var mock = new Mock();
mock.SetupGet(s => s.ConfiguredEndpoint)
.Returns(CreateCurrentEndpoint("urn:current"));
mock.Setup(s => s.ReadAsync(
It.IsAny(),
It.IsAny(),
It.IsAny(),
It.Is>(r => r.Count == 5),
It.IsAny()))
.ReturnsAsync(new ReadResponse
{
Results =
[
new DataValue(new Variant(redundancySupport), StatusCodes.Good),
new DataValue(new Variant(serviceLevel), StatusCodes.Good),
redundantServers == null
? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
: CreateRedundantServerArrayValue(redundantServers),
serverUris == null
? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
: new DataValue(new Variant(new ArrayOf(serverUris)), StatusCodes.Good),
new DataValue(new Variant(DateTime.MinValue), StatusCodes.Good)
],
DiagnosticInfos = []
});
SetupSingleNodeRead(
mock,
VariableIds.Server_ServerRedundancy_CurrentServerId,
string.IsNullOrEmpty(currentServerId)
? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
: new DataValue(new Variant(currentServerId), StatusCodes.Good));
return mock;
}
private static void SetupSingleNodeRead(
Mock mock,
NodeId nodeId,
DataValue result)
{
mock.Setup(s => s.ReadAsync(
It.IsAny(),
It.IsAny(),
It.IsAny(),
It.Is>(r => r.Count == 1 && r[0].NodeId == nodeId),
It.IsAny()))
.ReturnsAsync(new ReadResponse
{
Results = [result],
DiagnosticInfos = []
});
}
private static void VerifyBatchedRedundancyRead(Mock mock)
{
mock.Verify(s => s.ReadAsync(
It.IsAny(),
It.IsAny(),
It.IsAny(),
It.Is>(r => r.Count == 5),
It.IsAny()), Times.Once);
mock.Verify(s => s.ReadAsync(
It.IsAny(),
It.IsAny(),
It.IsAny(),
It.Is>(r =>
r.Count == 1 &&
(r[0].NodeId == VariableIds.Server_ServerRedundancy_RedundantServerArray ||
r[0].NodeId == VariableIds.Server_ServerRedundancy_ServerUriArray)),
It.IsAny()), Times.Never);
}
private static DataValue CreateRedundantServerArrayValue(
RedundantServerDataType[] redundantServers)
{
ArrayOf extensionObjects =
Array.ConvertAll(redundantServers, s => new ExtensionObject(s));
return new DataValue(new Variant(extensionObjects), StatusCodes.Good);
}
private static Mock CreateMockSessionWithBadStatus()
{
var mock = new Mock();
mock.Setup(s => s.ReadAsync(
It.IsAny(),
It.IsAny(),
It.IsAny(),
It.Is>(r => r.Count == 5),
It.IsAny()))
.ReturnsAsync(new ReadResponse
{
Results =
[
new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
],
DiagnosticInfos = []
});
return mock;
}
private static Mock CreateMockSessionWithThrowingOptionalReads()
{
return CreateMockSession(
redundancySupport: (int)RedundancySupport.Hot,
serviceLevel: ServiceLevels.NoData);
}
private sealed class FixedTimeProvider : TimeProvider
{
public FixedTimeProvider(DateTime utcNow)
{
m_utcNow = utcNow;
}
public override DateTimeOffset GetUtcNow()
{
return m_utcNow;
}
private readonly DateTimeOffset m_utcNow;
}
}
}