/* ======================================================================== * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Time.Testing; using Moq; using NUnit.Framework; namespace Opc.Ua.Client.Tests.ManagedSession { /// /// Unit tests for . /// [TestFixture] [Category("Client")] [Category("ServerRedundancy")] public sealed class ServerRedundancyHandlerTests { private static readonly DateTime s_now = new(2026, 6, 27, 4, 0, 0, DateTimeKind.Utc); private Mock m_resolver = null!; private DefaultServerRedundancyHandler m_handler = null!; /// /// Creates a strict endpoint resolver mock and a redundancy handler with a fixed clock for each scenario. /// [SetUp] public void SetUp() { m_resolver = new Mock(MockBehavior.Strict); m_handler = new DefaultServerRedundancyHandler(m_resolver.Object, new FixedTimeProvider(s_now)); } /// /// Verifies that server redundancy support values map to the corresponding redundancy modes. /// [TestCase(0, RedundancySupport.None)] [TestCase(1, RedundancySupport.Cold)] [TestCase(2, RedundancySupport.Warm)] [TestCase(3, RedundancySupport.Hot)] [TestCase(4, RedundancySupport.Transparent)] [TestCase(5, RedundancySupport.HotAndMirrored)] public async Task FetchRedundancyInfoMapsRedundancySupportAsync( int value, RedundancySupport expected) { Mock mockSession = CreateMockSession( redundancySupport: value, serviceLevel: ServiceLevels.HealthyMinimum); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); Assert.That(info.Mode, Is.EqualTo(expected)); } [TestCase(RedundancySupport.Cold)] [TestCase(RedundancySupport.Warm)] public async Task BasicRedundancyReadRetainsUnresolvedPeersWithoutDiscoveryAsync(RedundancySupport mode) { Mock session = CreateMockSession( (int)mode, ServiceLevels.Maximum, serverUris: ["urn:offline"]); ServerRedundancyInfo snapshot = await ((IServerRedundancyEndpointCache)m_handler) .ReadRedundancyInfoAsync(session.Object, CancellationToken.None).ConfigureAwait(false); Assert.That(snapshot.Mode, Is.EqualTo(mode)); Assert.That(snapshot.ServiceLevel, Is.EqualTo(ServiceLevels.Maximum)); Assert.That(snapshot.RedundantServers.Count, Is.EqualTo(1)); Assert.That(snapshot.RedundantServers[0].ServerUri, Is.EqualTo("urn:offline")); Assert.That(snapshot.RedundantServers[0].Endpoint, Is.Null); m_resolver.VerifyNoOtherCalls(); VerifyBatchedRedundancyRead(session); } [TestCase(false)] [TestCase(true)] public async Task PeerTimeoutPreservesOtherEndpointsAndCoalescesPendingDiscoveryAsync(bool ignoreCancellation) { var time = new FakeTimeProvider(); var handler = new DefaultServerRedundancyHandler(m_resolver.Object, time); ConfiguredEndpoint available = CreateEndpoint("urn:available", "opc.tcp://available:4840"); var blocked = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); var secondLookup = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); int lookups = 0; m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny())) .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) => { if (Interlocked.Increment(ref lookups) == 2) { secondLookup.TrySetResult(true); } return new ValueTask(ignoreCancellation ? blocked.Task : blocked.Task.WaitAsync(ct)); }); m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:available", It.IsAny(), It.IsAny())) .ReturnsAsync(available); Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline", "urn:available"]); var cache = (IServerRedundancyEndpointCache)handler; ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None) .ConfigureAwait(false); try { Task resolving = cache.ResolveCachedEndpointsAsync( basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask(); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:available", It.IsAny(), It.IsAny()), Times.Once); Assert.That(resolving.IsCompleted, Is.False); time.Advance(TimeSpan.FromSeconds(2)); ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(10)) .ConfigureAwait(false); Assert.That(resolved.Mode, Is.EqualTo(RedundancySupport.Cold)); Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null); Assert.That(resolved.RedundantServers[1].Endpoint, Is.SameAs(available)); Assert.That(handler.SelectFailoverTarget(resolved, session.Object.ConfiguredEndpoint), Is.SameAs(available)); Task retry = cache.ResolveCachedEndpointsAsync( resolved, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask(); if (!ignoreCancellation) { // The timed-out lookup drains on the thread pool, so the retry may first join it and only // start its own lookup once it has drained. Advancing the clock before then would expire // the retry's bound too, so wait for the fresh lookup. await secondLookup.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); } time.Advance(TimeSpan.FromSeconds(2)); await retry.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny()), Times.Exactly(ignoreCancellation ? 1 : 2)); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:available", It.IsAny(), It.IsAny()), Times.Once); } finally { blocked.TrySetResult(null); } } [Test] public async Task ConfiguredPeerDiscoveryTimeoutReplacesTheDefaultBoundAsync() { var time = new FakeTimeProvider(); var handler = new DefaultServerRedundancyHandler( m_resolver.Object, time, new ServerRedundancyOptions { PeerDiscoveryTimeout = TimeSpan.FromSeconds(30) }); var blocked = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny())) .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) => new ValueTask(blocked.Task.WaitAsync(ct))); Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]); var cache = (IServerRedundancyEndpointCache)handler; ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None) .ConfigureAwait(false); try { Task resolving = cache.ResolveCachedEndpointsAsync( basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask(); time.Advance(TimeSpan.FromSeconds(2)); await Task.Yield(); Assert.That(resolving.IsCompleted, Is.False); time.Advance(TimeSpan.FromSeconds(28)); ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(10)) .ConfigureAwait(false); Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null); } finally { blocked.TrySetResult(null); } } [Test] public void ServerRedundancyOptionsRejectNonPositiveBounds() { Assert.Multiple(() => { Assert.That( () => new ServerRedundancyOptions { RefreshTimeout = TimeSpan.Zero }.Validate(), Throws.InstanceOf()); Assert.That( () => new ServerRedundancyOptions { PeerDiscoveryTimeout = TimeSpan.FromSeconds(-5) }.Validate(), Throws.InstanceOf()); Assert.That( () => new ServerRedundancyOptions { RefreshTimeout = Timeout.InfiniteTimeSpan, PeerDiscoveryTimeout = Timeout.InfiniteTimeSpan }.Validate(), Throws.Nothing); Assert.That(new ServerRedundancyOptions().RefreshTimeout, Is.EqualTo(ServerRedundancyOptions.DefaultTimeout)); }); } [TestCase(false)] [TestCase(true)] public async Task PeerDiscoveryTimeoutCancelsResolverBeforeOwnedTimerFiresAsync(bool ignoreCancellation) { var clock = new FakeTimeProvider(); var timeProvider = new Mock { CallBase = true }; timeProvider.Setup(provider => provider.GetUtcNow()).Returns(clock.GetUtcNow); timeProvider.Setup(provider => provider.GetTimestamp()).Returns(clock.GetTimestamp); timeProvider.SetupGet(provider => provider.TimestampFrequency).Returns(clock.TimestampFrequency); int timerCount = 0; timeProvider.Setup(provider => provider.CreateTimer( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) => { if (Interlocked.Increment(ref timerCount) == 1) { // Deliver the outer wait's deadline before the owned cancellation timer. dueTime += TimeSpan.FromSeconds(1); } return clock.CreateTimer(callback, state, dueTime, period); }); var timeout = TimeSpan.FromSeconds(1); var handler = new DefaultServerRedundancyHandler( m_resolver.Object, timeProvider.Object, new ServerRedundancyOptions { PeerDiscoveryTimeout = timeout }); var blocked = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); CancellationToken discoveryToken = default; int resolverCalls = 0; ConfiguredEndpoint freshEndpoint = CreateEndpoint("urn:offline", "opc.tcp://fresh:4840"); m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny())) .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) => { if (Interlocked.Increment(ref resolverCalls) == 1) { discoveryToken = ct; return new ValueTask( ignoreCancellation ? blocked.Task : blocked.Task.WaitAsync(ct)); } return new ValueTask(freshEndpoint); }); Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]); var cache = (IServerRedundancyEndpointCache)handler; ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync( session.Object, CancellationToken.None).ConfigureAwait(false); try { Task resolving = cache.ResolveCachedEndpointsAsync( basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask(); Assert.That(timerCount, Is.EqualTo(2)); Assert.That(discoveryToken.CanBeCanceled, Is.True); clock.Advance(timeout); ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(5)) .ConfigureAwait(false); Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null); Assert.That(discoveryToken.IsCancellationRequested, Is.True); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny()), Times.Once); blocked.TrySetResult(CreateEndpoint("urn:offline", "opc.tcp://late:4840")); ServerRedundancyInfo retried = await cache.ResolveCachedEndpointsAsync( basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask() .WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); Assert.That(retried.RedundantServers[0].Endpoint, Is.SameAs(freshEndpoint)); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny()), Times.Exactly(2)); } finally { blocked.TrySetResult(null); } } [Test] public async Task PeerDiscoveryPropagatesCallerCancellationAsync() { using var cancellation = new CancellationTokenSource(); var resolverEntered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var blocked = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); CancellationToken discoveryToken = default; m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:offline", It.IsAny(), It.IsAny())) .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) => { discoveryToken = ct; resolverEntered.TrySetResult(true); return new ValueTask(blocked.Task.WaitAsync(ct)); }); Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]); var cache = (IServerRedundancyEndpointCache)m_handler; ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, cancellation.Token) .ConfigureAwait(false); Task resolving = cache.ResolveCachedEndpointsAsync( basic, session.Object.ConfiguredEndpoint, cancellation.Token).AsTask(); await resolverEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); cancellation.Cancel(); await Assert.ThatAsync( () => resolving, Throws.InstanceOf()).ConfigureAwait(false); Assert.That(discoveryToken.IsCancellationRequested, Is.True); Assert.That(basic.RedundantServers[0].Endpoint, Is.Null); } [Test] public async Task CancelledMetadataReadDoesNotStartPeerDiscoveryAsync() { using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]); await Assert.ThatAsync( async () => await m_handler.FetchRedundancyInfoAsync(session.Object, cancellation.Token) .ConfigureAwait(false), Throws.InstanceOf()).ConfigureAwait(false); session.VerifyNoOtherCalls(); m_resolver.VerifyNoOtherCalls(); } [Test] public async Task CachedResolutionDoesNotCrossEndpointSecurityProfilesAsync() { Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:peer"]); ConfiguredEndpoint unsecured = CreateEndpoint("urn:peer", "opc.tcp://peer:4840"); ConfiguredEndpoint secured = CreateEndpoint("urn:peer", "opc.tcp://peer:4840"); secured.Description.SecurityMode = MessageSecurityMode.Sign; secured.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256; m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:peer", It.Is(endpoint => endpoint.Description.SecurityMode == MessageSecurityMode.None), It.IsAny())) .ReturnsAsync(unsecured); m_resolver.Setup(resolver => resolver.ResolveAsync( "urn:peer", It.Is(endpoint => endpoint.Description.SecurityMode == MessageSecurityMode.Sign), It.IsAny())) .ReturnsAsync(secured); ServerRedundancyInfo first = await m_handler.FetchRedundancyInfoAsync(session.Object) .ConfigureAwait(false); Assert.That(first.RedundantServers[0].Endpoint, Is.SameAs(unsecured)); session.Object.ConfiguredEndpoint.Description.SecurityMode = MessageSecurityMode.Sign; session.Object.ConfiguredEndpoint.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256; ServerRedundancyInfo second = await m_handler.FetchRedundancyInfoAsync(session.Object) .ConfigureAwait(false); Assert.That(second.RedundantServers[0].Endpoint, Is.SameAs(secured)); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:peer", It.IsAny(), It.IsAny()), Times.Exactly(2)); } [Test] public async Task RetriedPeerDiscoveryRejectsLateCancelledResultAsync() { var time = new FakeTimeProvider(); var handler = new DefaultServerRedundancyHandler(m_resolver.Object, time); ConfiguredEndpoint stale = CreateEndpoint("urn:peer", "opc.tcp://peer:4840"); ConfiguredEndpoint current = CreateEndpoint("urn:peer", "opc.tcp://peer:4841"); var blocked = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); m_resolver.SetupSequence(resolver => resolver.ResolveAsync( "urn:peer", It.IsAny(), It.IsAny())) .Returns(new ValueTask(blocked.Task)) .ReturnsAsync(current); Mock session = CreateMockSession( (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:peer"]); var cache = (IServerRedundancyEndpointCache)handler; ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None) .ConfigureAwait(false); Task first = cache.ResolveCachedEndpointsAsync( basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask(); time.Advance(TimeSpan.FromSeconds(2)); ServerRedundancyInfo unresolved = await first.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); Assert.That(unresolved.RedundantServers[0].Endpoint, Is.Null); Task retry = cache.ResolveCachedEndpointsAsync( unresolved, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask(); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:peer", It.IsAny(), It.IsAny()), Times.Once); blocked.TrySetResult(stale); ServerRedundancyInfo resolved = await retry.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); Assert.That(resolved.RedundantServers[0].Endpoint, Is.SameAs(current)); ServerRedundancyInfo cached = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None) .ConfigureAwait(false); Assert.That(cached.RedundantServers[0].Endpoint, Is.SameAs(current)); m_resolver.Verify(resolver => resolver.ResolveAsync( "urn:peer", It.IsAny(), It.IsAny()), Times.Exactly(2)); } /// /// Verifies that a healthy server avoids proactive failover while retaining a recovery target. /// [Test] public void HealthyServerDoesNotProactivelyFailOverButRetainsRecoveryTarget() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.HealthyMinimum, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Healthy, RedundantServers = [ CreateServerInfo("urn:backup", ServiceLevels.Maximum, ServerState.Running) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That(target, Is.Not.Null); Assert.That( target!.Description.Server.ApplicationUri, Is.EqualTo("urn:backup")); } /// /// Verifies that an interface-typed call (the extension method) applies the /// same health checks as the concrete handler. /// [Test] public void InterfaceShouldFailoverDoesNotFailOverFromHealthyServer() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.HealthyMinimum, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Healthy, RedundantServers = [ CreateServerInfo("urn:backup", ServiceLevels.Maximum, ServerState.Running) ] }; IServerRedundancyHandler handler = m_handler; ServerFailoverDecision decision = handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); } /// /// Verifies that the extension fallback for a custom handler does not fail /// over from a Healthy server even when a target is available. /// [Test] public void CustomHandlerShouldFailoverDoesNotFailOverFromHealthyServer() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.Maximum, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Healthy }; var custom = new Mock(); custom.Setup(h => h.SelectFailoverTarget(It.IsAny(), It.IsAny())) .Returns(CreateCurrentEndpoint("urn:backup")); ServerFailoverDecision healthy = custom.Object.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); ServerFailoverDecision degraded = custom.Object.ShouldFailover( new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.DegradedMaximum, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Degraded }, CreateCurrentEndpoint("urn:current")); Assert.That(healthy.IsFailoverWarranted, Is.False); Assert.That(degraded.IsFailoverWarranted, Is.True); } /// /// Verifies that a degraded server fails over to a healthy peer. /// [Test] public void ShouldFailoverSwitchesFromDegradedToHealthyPeer() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.DegradedMaximum, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Degraded, RedundantServers = [ CreateServerInfo("urn:degraded", 150, ServerState.Running), CreateServerInfo("urn:healthy", 230, ServerState.Running) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.True); Assert.That(target, Is.Not.Null); Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy")); } /// /// Verifies that a degraded server does not fail over when no healthy peer is available. /// [Test] public void ShouldFailoverDoesNotSwitchFromDegradedWithoutHealthyPeer() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = 100, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Degraded, RedundantServers = [ CreateServerInfo("urn:degraded", 150, ServerState.Running) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); } /// /// Verifies that a server in maintenance fails over to an operational peer. /// [Test] public void ShouldFailoverSwitchesFromMaintenanceToOperationalPeer() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.Maintenance, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, EstimatedReturnTime = s_now.AddMinutes(10), RedundantServers = [ CreateServerInfo("urn:healthy", 230, ServerState.Running) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.True); Assert.That(target, Is.Not.Null); Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy")); } /// /// Verifies that failover timing honors a maintenance server's estimated return time. /// [Test] public void ShouldFailoverHonorsMaintenanceEstimatedReturnTime() { DateTime estimatedReturnTime = s_now.AddMinutes(10); var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.Maintenance, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, EstimatedReturnTime = estimatedReturnTime, RedundantServers = [ CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That(decision.RetryAfter, Is.EqualTo(estimatedReturnTime)); } /// /// Verifies that failover uses backoff after the maintenance return estimate has elapsed. /// [Test] public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeLapsed() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.Maintenance, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, EstimatedReturnTime = s_now.AddMinutes(-1), RedundantServers = [ CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That( decision.RetryAfter, Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff))); } /// /// Verifies that failover uses backoff when no maintenance return estimate is available. /// [Test] public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeIsAbsent() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.Maintenance, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.Maintenance, EstimatedReturnTime = DateTime.MinValue, RedundantServers = [ CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That( decision.RetryAfter, Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff))); } /// /// Verifies that no failover is requested when all peers are down. /// [Test] public void ShouldFailoverReturnsNoFailoverWhenAllPeersAreDown() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.NoData, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ CreateServerInfo("urn:shutdown", ServiceLevels.Maximum, ServerState.Shutdown), CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended) ] }; ServerFailoverDecision decision = m_handler.ShouldFailover( info, CreateCurrentEndpoint("urn:current")); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(decision.IsFailoverWarranted, Is.False); Assert.That(target, Is.Null); } /// /// Verifies that transparent redundancy does not select a client-side failover target. /// [Test] public void SelectFailoverTargetReturnsNullForTransparentMode() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Transparent, ServiceLevel = ServiceLevels.NoData, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ CreateServerInfo("urn:backup", 230, ServerState.Running) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Null); } /// /// Verifies that disabled redundancy does not select a failover target. /// [Test] public void SelectFailoverTargetReturnsNullForNoneMode() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.None, ServiceLevel = ServiceLevels.NoData, ServiceLevelAccessible = false, ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ CreateServerInfo("urn:backup", 230, ServerState.Running) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Null); } /// /// Verifies that failover target selection excludes the current endpoint. /// [Test] public void SelectFailoverTargetSkipsCurrentEndpoint() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.NoData, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ CreateServerInfo("urn:current", ServiceLevels.Maximum, ServerState.Running), CreateServerInfo("urn:backup", 230, ServerState.Running) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Not.Null); Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:backup")); } /// /// Verifies that failover target selection excludes servers that are not running. /// [Test] public void SelectFailoverTargetSkipsNonRunningServers() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.NoData, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended), CreateServerInfo("urn:shutdown", 240, ServerState.Shutdown), CreateServerInfo("urn:running", 230, ServerState.Running) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Not.Null); Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:running")); } /// /// Verifies that target selection returns null when no viable server remains. /// [Test] public void SelectFailoverTargetReturnsNullWhenNoViableServers() { var info = new ServerRedundancyInfo { Mode = RedundancySupport.Hot, ServiceLevel = ServiceLevels.NoData, ServiceLevelAccessible = true, ServiceLevelSubrange = ServiceLevelSubrange.NoData, RedundantServers = [ CreateServerInfo("urn:current", 230, ServerState.Running), CreateServerInfo("urn:down", 230, ServerState.Shutdown) ] }; ConfiguredEndpoint? result = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(result, Is.Null); } /// /// Verifies that redundancy discovery reads the redundant server array. /// [Test] public async Task FetchRedundancyInfoReadsRedundantServerArrayAsync() { var serverData = new RedundantServerDataType { ServerId = "urn:server1", ServiceLevel = 230, ServerState = ServerState.Running }; ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840"); m_resolver.Setup(r => r.ResolveAsync( "urn:server1", It.IsAny(), It.IsAny())) .ReturnsAsync(resolvedEndpoint); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: 100, redundantServers: [serverData]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot)); Assert.That(info.ServiceLevelSubrange, Is.EqualTo(ServiceLevelSubrange.Degraded)); Assert.That(info.RedundantServers, Has.Count.EqualTo(1)); Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server1")); Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint)); } /// /// Verifies that redundancy discovery resolves server application URIs to endpoints. /// [Test] public async Task FetchRedundancyInfoResolvesServerUriArrayToEndpointsAsync() { ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840"); m_resolver.Setup(r => r.ResolveAsync( "urn:server-uri", It.IsAny(), It.IsAny())) .ReturnsAsync(resolvedEndpoint); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: ServiceLevels.NoData, serverUris: ["urn:server-uri"]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); Assert.That(info.RedundantServers, Has.Count.EqualTo(1)); Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server-uri")); Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint)); Assert.That( info.RedundantServers[0].Endpoint!.Description.EndpointUrl, Is.EqualTo("opc.tcp://server-uri:4840")); VerifyBatchedRedundancyRead(mockSession); } /// /// Verifies that peers with unresolved server URIs are excluded from failover selection. /// [Test] public async Task FetchRedundancyInfoExcludesUnresolvedServerUriFromSelectionAsync() { m_resolver.Setup(r => r.ResolveAsync( "urn:unresolved", It.IsAny(), It.IsAny())) .ReturnsAsync((ConfiguredEndpoint?)null); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: ServiceLevels.NoData, serverUris: ["urn:unresolved"]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(info.RedundantServers, Has.Count.EqualTo(1)); Assert.That(info.RedundantServers[0].Endpoint, Is.Null); Assert.That(target, Is.Null); } /// /// Verifies that a URI-only peer can be selected when its service level is unknown. /// [Test] public async Task FetchRedundancyInfoSelectsServerUriOnlyPeerWithUnknownServiceLevelAsync() { ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840"); m_resolver.Setup(r => r.ResolveAsync( "urn:server-uri", It.IsAny(), It.IsAny())) .ReturnsAsync(resolvedEndpoint); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: ServiceLevels.NoData, serverUris: ["urn:server-uri"]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(info.RedundantServers[0].ServiceLevel, Is.EqualTo(ServiceLevels.NoData)); Assert.That(target, Is.SameAs(resolvedEndpoint)); } /// /// Verifies that discovery can fall back to a URI-only peer with unknown service level. /// [Test] public async Task FetchRedundancyInfoFallsBackToUnknownServerUriOnlyPeerAsync() { var serverData = new RedundantServerDataType { ServerId = "urn:nodata", ServiceLevel = ServiceLevels.NoData, ServerState = ServerState.Running }; ConfiguredEndpoint noDataEndpoint = CreateEndpoint("urn:nodata", "opc.tcp://nodata:4840"); ConfiguredEndpoint unknownEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840"); m_resolver.Setup(r => r.ResolveAsync( "urn:nodata", It.IsAny(), It.IsAny())) .ReturnsAsync(noDataEndpoint); m_resolver.Setup(r => r.ResolveAsync( "urn:server-uri", It.IsAny(), It.IsAny())) .ReturnsAsync(unknownEndpoint); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: ServiceLevels.DegradedMaximum, redundantServers: [serverData], serverUris: ["urn:nodata", "urn:server-uri"]); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); ConfiguredEndpoint? target = m_handler.SelectFailoverTarget( info, CreateCurrentEndpoint("urn:current")); Assert.That(info.RedundantServers, Has.Count.EqualTo(2)); Assert.That(target, Is.SameAs(unknownEndpoint)); } /// /// Verifies that redundancy discovery caches resolved peer endpoints. /// [Test] public async Task FetchRedundancyInfoCachesResolvedEndpointsAsync() { var serverData = new RedundantServerDataType { ServerId = "urn:server1", ServiceLevel = 230, ServerState = ServerState.Running }; ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840"); m_resolver.Setup(r => r.ResolveAsync( "urn:server1", It.IsAny(), It.IsAny())) .ReturnsAsync(resolvedEndpoint); Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: 100, redundantServers: [serverData]); await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false); await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false); m_resolver.Verify(r => r.ResolveAsync( "urn:server1", It.IsAny(), It.IsAny()), Times.Once); } /// /// Verifies that transparent redundancy discovery reads the current server identifier. /// [Test] public async Task FetchRedundancyInfoReadsTransparentCurrentServerIdAsync() { Mock mockSession = CreateMockSession( redundancySupport: (int)RedundancySupport.Transparent, serviceLevel: ServiceLevels.HealthyMinimum, currentServerId: "server-a"); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); Assert.That(info.CurrentServerId, Is.EqualTo("server-a")); } /// /// Verifies that failed redundancy reads report no redundancy, an inaccessible service level, and no peers. /// [Test] public async Task FetchRedundancyInfoHandlesReadErrorsAsync() { Mock mockSession = CreateMockSessionWithBadStatus(); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); Assert.That(info.Mode, Is.EqualTo(RedundancySupport.None)); Assert.That(info.ServiceLevel, Is.Zero); Assert.That(info.ServiceLevelAccessible, Is.False); Assert.That(info.RedundantServers, Is.Empty); } /// /// Verifies that malformed optional redundancy nodes preserve the discovered mode without inventing peers. /// [Test] public async Task FetchRedundancyInfoHandlesMalformedOptionalNodesAsync() { Mock mockSession = CreateMockSessionWithThrowingOptionalReads(); ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync( mockSession.Object).ConfigureAwait(false); Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot)); Assert.That(info.RedundantServers, Is.Empty); } /// /// Verifies that discovery and failover APIs reject null sessions, redundancy information, and endpoints. /// [Test] public void NullArgumentsThrow() { var info = new ServerRedundancyInfo(); ConfiguredEndpoint endpoint = CreateCurrentEndpoint("urn:current"); Assert.That( async () => await m_handler.FetchRedundancyInfoAsync(null!).ConfigureAwait(false), Throws.ArgumentNullException); Assert.That(() => m_handler.ShouldFailover(null!, endpoint), Throws.ArgumentNullException); Assert.That(() => m_handler.ShouldFailover(info, null!), Throws.ArgumentNullException); Assert.That(() => m_handler.SelectFailoverTarget(null!, endpoint), Throws.ArgumentNullException); Assert.That(() => m_handler.SelectFailoverTarget(info, null!), Throws.ArgumentNullException); } private static RedundantServer CreateServerInfo( string uri, byte serviceLevel, ServerState state) { return new RedundantServer { ServerUri = uri, ServiceLevel = serviceLevel, ServerState = state, Endpoint = CreateEndpoint(uri, $"opc.tcp://{uri[4..]}:4840") }; } private static ConfiguredEndpoint CreateCurrentEndpoint(string applicationUri) { return CreateEndpoint(applicationUri, "opc.tcp://current:4840"); } private static ConfiguredEndpoint CreateEndpoint(string applicationUri, string endpointUrl) { var description = new EndpointDescription { EndpointUrl = endpointUrl, SecurityMode = MessageSecurityMode.None, SecurityPolicyUri = SecurityPolicies.None, Server = new ApplicationDescription { ApplicationUri = applicationUri, DiscoveryUrls = new ArrayOf(new[] { endpointUrl }) } }; return new ConfiguredEndpoint(null, description, configuration: null); } private static Mock CreateMockSession( int redundancySupport, byte serviceLevel, RedundantServerDataType[]? redundantServers = null, string[]? serverUris = null, string currentServerId = "") { var mock = new Mock(); mock.SetupGet(s => s.ConfiguredEndpoint) .Returns(CreateCurrentEndpoint("urn:current")); mock.Setup(s => s.ReadAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.Is>(r => r.Count == 5), It.IsAny())) .ReturnsAsync(new ReadResponse { Results = [ new DataValue(new Variant(redundancySupport), StatusCodes.Good), new DataValue(new Variant(serviceLevel), StatusCodes.Good), redundantServers == null ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) : CreateRedundantServerArrayValue(redundantServers), serverUris == null ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) : new DataValue(new Variant(new ArrayOf(serverUris)), StatusCodes.Good), new DataValue(new Variant(DateTime.MinValue), StatusCodes.Good) ], DiagnosticInfos = [] }); SetupSingleNodeRead( mock, VariableIds.Server_ServerRedundancy_CurrentServerId, string.IsNullOrEmpty(currentServerId) ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) : new DataValue(new Variant(currentServerId), StatusCodes.Good)); return mock; } private static void SetupSingleNodeRead( Mock mock, NodeId nodeId, DataValue result) { mock.Setup(s => s.ReadAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.Is>(r => r.Count == 1 && r[0].NodeId == nodeId), It.IsAny())) .ReturnsAsync(new ReadResponse { Results = [result], DiagnosticInfos = [] }); } private static void VerifyBatchedRedundancyRead(Mock mock) { mock.Verify(s => s.ReadAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.Is>(r => r.Count == 5), It.IsAny()), Times.Once); mock.Verify(s => s.ReadAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.Is>(r => r.Count == 1 && (r[0].NodeId == VariableIds.Server_ServerRedundancy_RedundantServerArray || r[0].NodeId == VariableIds.Server_ServerRedundancy_ServerUriArray)), It.IsAny()), Times.Never); } private static DataValue CreateRedundantServerArrayValue( RedundantServerDataType[] redundantServers) { ArrayOf extensionObjects = Array.ConvertAll(redundantServers, s => new ExtensionObject(s)); return new DataValue(new Variant(extensionObjects), StatusCodes.Good); } private static Mock CreateMockSessionWithBadStatus() { var mock = new Mock(); mock.Setup(s => s.ReadAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.Is>(r => r.Count == 5), It.IsAny())) .ReturnsAsync(new ReadResponse { Results = [ new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown), new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown) ], DiagnosticInfos = [] }); return mock; } private static Mock CreateMockSessionWithThrowingOptionalReads() { return CreateMockSession( redundancySupport: (int)RedundancySupport.Hot, serviceLevel: ServiceLevels.NoData); } private sealed class FixedTimeProvider : TimeProvider { public FixedTimeProvider(DateTime utcNow) { m_utcNow = utcNow; } public override DateTimeOffset GetUtcNow() { return m_utcNow; } private readonly DateTimeOffset m_utcNow; } } }