UA-.NETStandard/tests/Opc.Ua.Client.Tests/Session/ServerRedundancyHandlerTests.cs

/* ========================================================================
 * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved.
 *
 * OPC Foundation MIT License 1.00
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * The complete license agreement can be found here:
 * http://opcfoundation.org/License/MIT/1.00/
 * ======================================================================*/

using System;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Time.Testing;
using Moq;
using NUnit.Framework;

namespace Opc.Ua.Client.Tests.ManagedSession
{
    /// <summary>
    /// Unit tests for <see cref="DefaultServerRedundancyHandler"/>.
    /// </summary>
    [TestFixture]
    [Category("Client")]
    [Category("ServerRedundancy")]
    public sealed class ServerRedundancyHandlerTests
    {
        private static readonly DateTime s_now = new(2026, 6, 27, 4, 0, 0, DateTimeKind.Utc);
        private Mock<IRedundantServerEndpointResolver> m_resolver = null!;
        private DefaultServerRedundancyHandler m_handler = null!;

        /// <summary>
        /// Creates a strict endpoint resolver mock and a redundancy handler with a fixed clock for each scenario.
        /// </summary>
        [SetUp]
        public void SetUp()
        {
            m_resolver = new Mock<IRedundantServerEndpointResolver>(MockBehavior.Strict);
            m_handler = new DefaultServerRedundancyHandler(m_resolver.Object, new FixedTimeProvider(s_now));
        }

        /// <summary>
        /// Verifies that server redundancy support values map to the corresponding redundancy modes.
        /// </summary>
        [TestCase(0, RedundancySupport.None)]
        [TestCase(1, RedundancySupport.Cold)]
        [TestCase(2, RedundancySupport.Warm)]
        [TestCase(3, RedundancySupport.Hot)]
        [TestCase(4, RedundancySupport.Transparent)]
        [TestCase(5, RedundancySupport.HotAndMirrored)]
        public async Task FetchRedundancyInfoMapsRedundancySupportAsync(
            int value,
            RedundancySupport expected)
        {
            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: value,
                serviceLevel: ServiceLevels.HealthyMinimum);

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);

            Assert.That(info.Mode, Is.EqualTo(expected));
        }

        [TestCase(RedundancySupport.Cold)]
        [TestCase(RedundancySupport.Warm)]
        public async Task BasicRedundancyReadRetainsUnresolvedPeersWithoutDiscoveryAsync(RedundancySupport mode)
        {
            Mock<ISession> session = CreateMockSession(
                (int)mode, ServiceLevels.Maximum, serverUris: ["urn:offline"]);

            ServerRedundancyInfo snapshot = await ((IServerRedundancyEndpointCache)m_handler)
                .ReadRedundancyInfoAsync(session.Object, CancellationToken.None).ConfigureAwait(false);

            Assert.That(snapshot.Mode, Is.EqualTo(mode));
            Assert.That(snapshot.ServiceLevel, Is.EqualTo(ServiceLevels.Maximum));
            Assert.That(snapshot.RedundantServers.Count, Is.EqualTo(1));
            Assert.That(snapshot.RedundantServers[0].ServerUri, Is.EqualTo("urn:offline"));
            Assert.That(snapshot.RedundantServers[0].Endpoint, Is.Null);
            m_resolver.VerifyNoOtherCalls();
            VerifyBatchedRedundancyRead(session);
        }

        [TestCase(false)]
        [TestCase(true)]
        public async Task PeerTimeoutPreservesOtherEndpointsAndCoalescesPendingDiscoveryAsync(bool ignoreCancellation)
        {
            var time = new FakeTimeProvider();
            var handler = new DefaultServerRedundancyHandler(m_resolver.Object, time);
            ConfiguredEndpoint available = CreateEndpoint("urn:available", "opc.tcp://available:4840");
            var blocked = new TaskCompletionSource<ConfiguredEndpoint?>(
                TaskCreationOptions.RunContinuationsAsynchronously);
            var secondLookup = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            int lookups = 0;
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()))
                .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
                {
                    if (Interlocked.Increment(ref lookups) == 2)
                    {
                        secondLookup.TrySetResult(true);
                    }
                    return new ValueTask<ConfiguredEndpoint?>(ignoreCancellation
                        ? blocked.Task
                        : blocked.Task.WaitAsync(ct));
                });
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:available", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()))
                .ReturnsAsync(available);
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline", "urn:available"]);
            var cache = (IServerRedundancyEndpointCache)handler;
            ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
                .ConfigureAwait(false);
            try
            {
                Task<ServerRedundancyInfo> resolving = cache.ResolveCachedEndpointsAsync(
                    basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
                m_resolver.Verify(resolver => resolver.ResolveAsync(
                    "urn:available", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Once);
                Assert.That(resolving.IsCompleted, Is.False);
                time.Advance(TimeSpan.FromSeconds(2));

                ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(10))
                    .ConfigureAwait(false);

                Assert.That(resolved.Mode, Is.EqualTo(RedundancySupport.Cold));
                Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null);
                Assert.That(resolved.RedundantServers[1].Endpoint, Is.SameAs(available));
                Assert.That(handler.SelectFailoverTarget(resolved, session.Object.ConfiguredEndpoint),
                    Is.SameAs(available));

                Task<ServerRedundancyInfo> retry = cache.ResolveCachedEndpointsAsync(
                    resolved, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
                if (!ignoreCancellation)
                {
                    // The timed-out lookup drains on the thread pool, so the retry may first join it and only
                    // start its own lookup once it has drained. Advancing the clock before then would expire
                    // the retry's bound too, so wait for the fresh lookup.
                    await secondLookup.Task.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
                }
                time.Advance(TimeSpan.FromSeconds(2));
                await retry.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
                m_resolver.Verify(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()),
                    Times.Exactly(ignoreCancellation ? 1 : 2));
                m_resolver.Verify(resolver => resolver.ResolveAsync(
                    "urn:available", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Once);
            }
            finally
            {
                blocked.TrySetResult(null);
            }
        }

        [Test]
        public async Task ConfiguredPeerDiscoveryTimeoutReplacesTheDefaultBoundAsync()
        {
            var time = new FakeTimeProvider();
            var handler = new DefaultServerRedundancyHandler(
                m_resolver.Object,
                time,
                new ServerRedundancyOptions { PeerDiscoveryTimeout = TimeSpan.FromSeconds(30) });
            var blocked = new TaskCompletionSource<ConfiguredEndpoint?>(
                TaskCreationOptions.RunContinuationsAsynchronously);
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()))
                .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
                    new ValueTask<ConfiguredEndpoint?>(blocked.Task.WaitAsync(ct)));
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
            var cache = (IServerRedundancyEndpointCache)handler;
            ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
                .ConfigureAwait(false);
            try
            {
                Task<ServerRedundancyInfo> resolving = cache.ResolveCachedEndpointsAsync(
                    basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();

                time.Advance(TimeSpan.FromSeconds(2));
                await Task.Yield();
                Assert.That(resolving.IsCompleted, Is.False);

                time.Advance(TimeSpan.FromSeconds(28));
                ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(10))
                    .ConfigureAwait(false);

                Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null);
            }
            finally
            {
                blocked.TrySetResult(null);
            }
        }

        [Test]
        public void ServerRedundancyOptionsRejectNonPositiveBounds()
        {
            Assert.Multiple(() =>
            {
                Assert.That(
                    () => new ServerRedundancyOptions { RefreshTimeout = TimeSpan.Zero }.Validate(),
                    Throws.InstanceOf<ArgumentOutOfRangeException>());
                Assert.That(
                    () => new ServerRedundancyOptions
                    {
                        PeerDiscoveryTimeout = TimeSpan.FromSeconds(-5)
                    }.Validate(),
                    Throws.InstanceOf<ArgumentOutOfRangeException>());
                Assert.That(
                    () => new ServerRedundancyOptions
                    {
                        RefreshTimeout = Timeout.InfiniteTimeSpan,
                        PeerDiscoveryTimeout = Timeout.InfiniteTimeSpan
                    }.Validate(),
                    Throws.Nothing);
                Assert.That(new ServerRedundancyOptions().RefreshTimeout,
                    Is.EqualTo(ServerRedundancyOptions.DefaultTimeout));
            });
        }

        [TestCase(false)]
        [TestCase(true)]
        public async Task PeerDiscoveryTimeoutCancelsResolverBeforeOwnedTimerFiresAsync(bool ignoreCancellation)
        {
            var clock = new FakeTimeProvider();
            var timeProvider = new Mock<TimeProvider> { CallBase = true };
            timeProvider.Setup(provider => provider.GetUtcNow()).Returns(clock.GetUtcNow);
            timeProvider.Setup(provider => provider.GetTimestamp()).Returns(clock.GetTimestamp);
            timeProvider.SetupGet(provider => provider.TimestampFrequency).Returns(clock.TimestampFrequency);
            int timerCount = 0;
            timeProvider.Setup(provider => provider.CreateTimer(
                    It.IsAny<TimerCallback>(), It.IsAny<object?>(),
                    It.IsAny<TimeSpan>(), It.IsAny<TimeSpan>()))
                .Returns((TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) =>
                {
                    if (Interlocked.Increment(ref timerCount) == 1)
                    {
                        // Deliver the outer wait's deadline before the owned cancellation timer.
                        dueTime += TimeSpan.FromSeconds(1);
                    }
                    return clock.CreateTimer(callback, state, dueTime, period);
                });
            var timeout = TimeSpan.FromSeconds(1);
            var handler = new DefaultServerRedundancyHandler(
                m_resolver.Object, timeProvider.Object,
                new ServerRedundancyOptions { PeerDiscoveryTimeout = timeout });
            var blocked = new TaskCompletionSource<ConfiguredEndpoint?>(
                TaskCreationOptions.RunContinuationsAsynchronously);
            CancellationToken discoveryToken = default;
            int resolverCalls = 0;
            ConfiguredEndpoint freshEndpoint = CreateEndpoint("urn:offline", "opc.tcp://fresh:4840");
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()))
                .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
                {
                    if (Interlocked.Increment(ref resolverCalls) == 1)
                    {
                        discoveryToken = ct;
                        return new ValueTask<ConfiguredEndpoint?>(
                            ignoreCancellation ? blocked.Task : blocked.Task.WaitAsync(ct));
                    }
                    return new ValueTask<ConfiguredEndpoint?>(freshEndpoint);
                });
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
            var cache = (IServerRedundancyEndpointCache)handler;
            ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(
                session.Object, CancellationToken.None).ConfigureAwait(false);
            try
            {
                Task<ServerRedundancyInfo> resolving = cache.ResolveCachedEndpointsAsync(
                    basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
                Assert.That(timerCount, Is.EqualTo(2));
                Assert.That(discoveryToken.CanBeCanceled, Is.True);

                clock.Advance(timeout);
                ServerRedundancyInfo resolved = await resolving.WaitAsync(TimeSpan.FromSeconds(5))
                    .ConfigureAwait(false);

                Assert.That(resolved.RedundantServers[0].Endpoint, Is.Null);
                Assert.That(discoveryToken.IsCancellationRequested, Is.True);
                m_resolver.Verify(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Once);

                blocked.TrySetResult(CreateEndpoint("urn:offline", "opc.tcp://late:4840"));
                ServerRedundancyInfo retried = await cache.ResolveCachedEndpointsAsync(
                    basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask()
                    .WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
                Assert.That(retried.RedundantServers[0].Endpoint, Is.SameAs(freshEndpoint));
                m_resolver.Verify(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
            }
            finally
            {
                blocked.TrySetResult(null);
            }
        }

        [Test]
        public async Task PeerDiscoveryPropagatesCallerCancellationAsync()
        {
            using var cancellation = new CancellationTokenSource();
            var resolverEntered = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
            var blocked = new TaskCompletionSource<ConfiguredEndpoint?>(
                TaskCreationOptions.RunContinuationsAsynchronously);
            CancellationToken discoveryToken = default;
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:offline", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()))
                .Returns((string _, ConfiguredEndpoint _, CancellationToken ct) =>
                {
                    discoveryToken = ct;
                    resolverEntered.TrySetResult(true);
                    return new ValueTask<ConfiguredEndpoint?>(blocked.Task.WaitAsync(ct));
                });
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);
            var cache = (IServerRedundancyEndpointCache)m_handler;
            ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, cancellation.Token)
                .ConfigureAwait(false);
            Task<ServerRedundancyInfo> resolving = cache.ResolveCachedEndpointsAsync(
                basic, session.Object.ConfiguredEndpoint, cancellation.Token).AsTask();

            await resolverEntered.Task.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
            cancellation.Cancel();

            await Assert.ThatAsync(
                () => resolving,
                Throws.InstanceOf<OperationCanceledException>()).ConfigureAwait(false);
            Assert.That(discoveryToken.IsCancellationRequested, Is.True);
            Assert.That(basic.RedundantServers[0].Endpoint, Is.Null);
        }

        [Test]
        public async Task CancelledMetadataReadDoesNotStartPeerDiscoveryAsync()
        {
            using var cancellation = new CancellationTokenSource();
            cancellation.Cancel();
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:offline"]);

            await Assert.ThatAsync(
                async () => await m_handler.FetchRedundancyInfoAsync(session.Object, cancellation.Token)
                    .ConfigureAwait(false),
                Throws.InstanceOf<OperationCanceledException>()).ConfigureAwait(false);

            session.VerifyNoOtherCalls();
            m_resolver.VerifyNoOtherCalls();
        }

        [Test]
        public async Task CachedResolutionDoesNotCrossEndpointSecurityProfilesAsync()
        {
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:peer"]);
            ConfiguredEndpoint unsecured = CreateEndpoint("urn:peer", "opc.tcp://peer:4840");
            ConfiguredEndpoint secured = CreateEndpoint("urn:peer", "opc.tcp://peer:4840");
            secured.Description.SecurityMode = MessageSecurityMode.Sign;
            secured.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256;
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:peer", It.Is<ConfiguredEndpoint>(endpoint =>
                        endpoint.Description.SecurityMode == MessageSecurityMode.None),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(unsecured);
            m_resolver.Setup(resolver => resolver.ResolveAsync(
                    "urn:peer", It.Is<ConfiguredEndpoint>(endpoint =>
                        endpoint.Description.SecurityMode == MessageSecurityMode.Sign),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(secured);
            ServerRedundancyInfo first = await m_handler.FetchRedundancyInfoAsync(session.Object)
                .ConfigureAwait(false);
            Assert.That(first.RedundantServers[0].Endpoint, Is.SameAs(unsecured));
            session.Object.ConfiguredEndpoint.Description.SecurityMode = MessageSecurityMode.Sign;
            session.Object.ConfiguredEndpoint.Description.SecurityPolicyUri = SecurityPolicies.Basic256Sha256;

            ServerRedundancyInfo second = await m_handler.FetchRedundancyInfoAsync(session.Object)
                .ConfigureAwait(false);

            Assert.That(second.RedundantServers[0].Endpoint, Is.SameAs(secured));
            m_resolver.Verify(resolver => resolver.ResolveAsync(
                "urn:peer", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
        }

        [Test]
        public async Task RetriedPeerDiscoveryRejectsLateCancelledResultAsync()
        {
            var time = new FakeTimeProvider();
            var handler = new DefaultServerRedundancyHandler(m_resolver.Object, time);
            ConfiguredEndpoint stale = CreateEndpoint("urn:peer", "opc.tcp://peer:4840");
            ConfiguredEndpoint current = CreateEndpoint("urn:peer", "opc.tcp://peer:4841");
            var blocked = new TaskCompletionSource<ConfiguredEndpoint?>(
                TaskCreationOptions.RunContinuationsAsynchronously);
            m_resolver.SetupSequence(resolver => resolver.ResolveAsync(
                    "urn:peer", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()))
                .Returns(new ValueTask<ConfiguredEndpoint?>(blocked.Task))
                .ReturnsAsync(current);
            Mock<ISession> session = CreateMockSession(
                (int)RedundancySupport.Cold, ServiceLevels.Maximum, serverUris: ["urn:peer"]);
            var cache = (IServerRedundancyEndpointCache)handler;
            ServerRedundancyInfo basic = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
                .ConfigureAwait(false);
            Task<ServerRedundancyInfo> first = cache.ResolveCachedEndpointsAsync(
                basic, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
            time.Advance(TimeSpan.FromSeconds(2));
            ServerRedundancyInfo unresolved = await first.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
            Assert.That(unresolved.RedundantServers[0].Endpoint, Is.Null);
            Task<ServerRedundancyInfo> retry = cache.ResolveCachedEndpointsAsync(
                unresolved, session.Object.ConfiguredEndpoint, CancellationToken.None).AsTask();
            m_resolver.Verify(resolver => resolver.ResolveAsync(
                "urn:peer", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Once);

            blocked.TrySetResult(stale);
            ServerRedundancyInfo resolved = await retry.WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);

            Assert.That(resolved.RedundantServers[0].Endpoint, Is.SameAs(current));
            ServerRedundancyInfo cached = await cache.ReadRedundancyInfoAsync(session.Object, CancellationToken.None)
                .ConfigureAwait(false);
            Assert.That(cached.RedundantServers[0].Endpoint, Is.SameAs(current));
            m_resolver.Verify(resolver => resolver.ResolveAsync(
                "urn:peer", It.IsAny<ConfiguredEndpoint>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
        }

        /// <summary>
        /// Verifies that a healthy server avoids proactive failover while retaining a recovery target.
        /// </summary>
        [Test]
        public void HealthyServerDoesNotProactivelyFailOverButRetainsRecoveryTarget()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.HealthyMinimum,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Healthy,
                RedundantServers =
                [
                    CreateServerInfo("urn:backup", ServiceLevels.Maximum, ServerState.Running)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.False);
            Assert.That(target, Is.Not.Null);
            Assert.That(
                target!.Description.Server.ApplicationUri,
                Is.EqualTo("urn:backup"));
        }

        /// <summary>
        /// Verifies that a degraded server fails over to a healthy peer.
        /// </summary>
        [Test]
        public void ShouldFailoverSwitchesFromDegradedToHealthyPeer()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.DegradedMaximum,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Degraded,
                RedundantServers =
                [
                    CreateServerInfo("urn:degraded", 150, ServerState.Running),
                    CreateServerInfo("urn:healthy", 230, ServerState.Running)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.True);
            Assert.That(target, Is.Not.Null);
            Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy"));
        }

        /// <summary>
        /// Verifies that a degraded server does not fail over when no healthy peer is available.
        /// </summary>
        [Test]
        public void ShouldFailoverDoesNotSwitchFromDegradedWithoutHealthyPeer()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = 100,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Degraded,
                RedundantServers =
                [
                    CreateServerInfo("urn:degraded", 150, ServerState.Running)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.False);
        }

        /// <summary>
        /// Verifies that a server in maintenance fails over to an operational peer.
        /// </summary>
        [Test]
        public void ShouldFailoverSwitchesFromMaintenanceToOperationalPeer()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.Maintenance,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
                EstimatedReturnTime = s_now.AddMinutes(10),
                RedundantServers =
                [
                    CreateServerInfo("urn:healthy", 230, ServerState.Running)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.True);
            Assert.That(target, Is.Not.Null);
            Assert.That(target!.Description.Server.ApplicationUri, Is.EqualTo("urn:healthy"));
        }

        /// <summary>
        /// Verifies that failover timing honors a maintenance server's estimated return time.
        /// </summary>
        [Test]
        public void ShouldFailoverHonorsMaintenanceEstimatedReturnTime()
        {
            DateTime estimatedReturnTime = s_now.AddMinutes(10);
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.Maintenance,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
                EstimatedReturnTime = estimatedReturnTime,
                RedundantServers =
                [
                    CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.False);
            Assert.That(decision.RetryAfter, Is.EqualTo(estimatedReturnTime));
        }

        /// <summary>
        /// Verifies that failover uses backoff after the maintenance return estimate has elapsed.
        /// </summary>
        [Test]
        public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeLapsed()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.Maintenance,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
                EstimatedReturnTime = s_now.AddMinutes(-1),
                RedundantServers =
                [
                    CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.False);
            Assert.That(
                decision.RetryAfter,
                Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff)));
        }

        /// <summary>
        /// Verifies that failover uses backoff when no maintenance return estimate is available.
        /// </summary>
        [Test]
        public void ShouldFailoverUsesBackoffWhenMaintenanceReturnTimeIsAbsent()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.Maintenance,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.Maintenance,
                EstimatedReturnTime = DateTime.MinValue,
                RedundantServers =
                [
                    CreateServerInfo("urn:shutdown", 230, ServerState.Shutdown)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.False);
            Assert.That(
                decision.RetryAfter,
                Is.EqualTo(s_now.Add(DefaultServerRedundancyHandler.DefaultMaintenanceBackoff)));
        }

        /// <summary>
        /// Verifies that no failover is requested when all peers are down.
        /// </summary>
        [Test]
        public void ShouldFailoverReturnsNoFailoverWhenAllPeersAreDown()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.NoData,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.NoData,
                RedundantServers =
                [
                    CreateServerInfo("urn:shutdown", ServiceLevels.Maximum, ServerState.Shutdown),
                    CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended)
                ]
            };

            ServerFailoverDecision decision = m_handler.ShouldFailover(
                info, CreateCurrentEndpoint("urn:current"));
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(decision.IsFailoverWarranted, Is.False);
            Assert.That(target, Is.Null);
        }

        /// <summary>
        /// Verifies that transparent redundancy does not select a client-side failover target.
        /// </summary>
        [Test]
        public void SelectFailoverTargetReturnsNullForTransparentMode()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Transparent,
                ServiceLevel = ServiceLevels.NoData,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.NoData,
                RedundantServers =
                [
                    CreateServerInfo("urn:backup", 230, ServerState.Running)
                ]
            };

            ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(result, Is.Null);
        }

        /// <summary>
        /// Verifies that disabled redundancy does not select a failover target.
        /// </summary>
        [Test]
        public void SelectFailoverTargetReturnsNullForNoneMode()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.None,
                ServiceLevel = ServiceLevels.NoData,
                ServiceLevelAccessible = false,
                ServiceLevelSubrange = ServiceLevelSubrange.NoData,
                RedundantServers =
                [
                    CreateServerInfo("urn:backup", 230, ServerState.Running)
                ]
            };

            ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(result, Is.Null);
        }

        /// <summary>
        /// Verifies that failover target selection excludes the current endpoint.
        /// </summary>
        [Test]
        public void SelectFailoverTargetSkipsCurrentEndpoint()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.NoData,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.NoData,
                RedundantServers =
                [
                    CreateServerInfo("urn:current", ServiceLevels.Maximum, ServerState.Running),
                    CreateServerInfo("urn:backup", 230, ServerState.Running)
                ]
            };

            ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(result, Is.Not.Null);
            Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:backup"));
        }

        /// <summary>
        /// Verifies that failover target selection excludes servers that are not running.
        /// </summary>
        [Test]
        public void SelectFailoverTargetSkipsNonRunningServers()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.NoData,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.NoData,
                RedundantServers =
                [
                    CreateServerInfo("urn:suspended", ServiceLevels.Maximum, ServerState.Suspended),
                    CreateServerInfo("urn:shutdown", 240, ServerState.Shutdown),
                    CreateServerInfo("urn:running", 230, ServerState.Running)
                ]
            };

            ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(result, Is.Not.Null);
            Assert.That(result!.Description.Server.ApplicationUri, Is.EqualTo("urn:running"));
        }

        /// <summary>
        /// Verifies that target selection returns null when no viable server remains.
        /// </summary>
        [Test]
        public void SelectFailoverTargetReturnsNullWhenNoViableServers()
        {
            var info = new ServerRedundancyInfo
            {
                Mode = RedundancySupport.Hot,
                ServiceLevel = ServiceLevels.NoData,
                ServiceLevelAccessible = true,
                ServiceLevelSubrange = ServiceLevelSubrange.NoData,
                RedundantServers =
                [
                    CreateServerInfo("urn:current", 230, ServerState.Running),
                    CreateServerInfo("urn:down", 230, ServerState.Shutdown)
                ]
            };

            ConfiguredEndpoint? result = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(result, Is.Null);
        }

        /// <summary>
        /// Verifies that redundancy discovery reads the redundant server array.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoReadsRedundantServerArrayAsync()
        {
            var serverData = new RedundantServerDataType
            {
                ServerId = "urn:server1",
                ServiceLevel = 230,
                ServerState = ServerState.Running
            };
            ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840");
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:server1",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(resolvedEndpoint);

            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: 100,
                redundantServers: [serverData]);

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);

            Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot));
            Assert.That(info.ServiceLevelSubrange, Is.EqualTo(ServiceLevelSubrange.Degraded));
            Assert.That(info.RedundantServers, Has.Count.EqualTo(1));
            Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server1"));
            Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint));
        }

        /// <summary>
        /// Verifies that redundancy discovery resolves server application URIs to endpoints.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoResolvesServerUriArrayToEndpointsAsync()
        {
            ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840");
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:server-uri",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(resolvedEndpoint);

            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: ServiceLevels.NoData,
                serverUris: ["urn:server-uri"]);

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);

            Assert.That(info.RedundantServers, Has.Count.EqualTo(1));
            Assert.That(info.RedundantServers[0].ServerUri, Is.EqualTo("urn:server-uri"));
            Assert.That(info.RedundantServers[0].Endpoint, Is.SameAs(resolvedEndpoint));
            Assert.That(
                info.RedundantServers[0].Endpoint!.Description.EndpointUrl,
                Is.EqualTo("opc.tcp://server-uri:4840"));
            VerifyBatchedRedundancyRead(mockSession);
        }

        /// <summary>
        /// Verifies that peers with unresolved server URIs are excluded from failover selection.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoExcludesUnresolvedServerUriFromSelectionAsync()
        {
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:unresolved",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync((ConfiguredEndpoint?)null);
            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: ServiceLevels.NoData,
                serverUris: ["urn:unresolved"]);

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(info.RedundantServers, Has.Count.EqualTo(1));
            Assert.That(info.RedundantServers[0].Endpoint, Is.Null);
            Assert.That(target, Is.Null);
        }

        /// <summary>
        /// Verifies that a URI-only peer can be selected when its service level is unknown.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoSelectsServerUriOnlyPeerWithUnknownServiceLevelAsync()
        {
            ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840");
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:server-uri",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(resolvedEndpoint);
            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: ServiceLevels.NoData,
                serverUris: ["urn:server-uri"]);

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(info.RedundantServers[0].ServiceLevel, Is.EqualTo(ServiceLevels.NoData));
            Assert.That(target, Is.SameAs(resolvedEndpoint));
        }

        /// <summary>
        /// Verifies that discovery can fall back to a URI-only peer with unknown service level.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoFallsBackToUnknownServerUriOnlyPeerAsync()
        {
            var serverData = new RedundantServerDataType
            {
                ServerId = "urn:nodata",
                ServiceLevel = ServiceLevels.NoData,
                ServerState = ServerState.Running
            };
            ConfiguredEndpoint noDataEndpoint = CreateEndpoint("urn:nodata", "opc.tcp://nodata:4840");
            ConfiguredEndpoint unknownEndpoint = CreateEndpoint("urn:server-uri", "opc.tcp://server-uri:4840");
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:nodata",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(noDataEndpoint);
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:server-uri",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(unknownEndpoint);
            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: ServiceLevels.DegradedMaximum,
                redundantServers: [serverData],
                serverUris: ["urn:nodata", "urn:server-uri"]);

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);
            ConfiguredEndpoint? target = m_handler.SelectFailoverTarget(
                info, CreateCurrentEndpoint("urn:current"));

            Assert.That(info.RedundantServers, Has.Count.EqualTo(2));
            Assert.That(target, Is.SameAs(unknownEndpoint));
        }

        /// <summary>
        /// Verifies that redundancy discovery caches resolved peer endpoints.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoCachesResolvedEndpointsAsync()
        {
            var serverData = new RedundantServerDataType
            {
                ServerId = "urn:server1",
                ServiceLevel = 230,
                ServerState = ServerState.Running
            };
            ConfiguredEndpoint resolvedEndpoint = CreateEndpoint("urn:server1", "opc.tcp://server1:4840");
            m_resolver.Setup(r => r.ResolveAsync(
                    "urn:server1",
                    It.IsAny<ConfiguredEndpoint>(),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(resolvedEndpoint);
            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: 100,
                redundantServers: [serverData]);

            await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false);
            await m_handler.FetchRedundancyInfoAsync(mockSession.Object).ConfigureAwait(false);

            m_resolver.Verify(r => r.ResolveAsync(
                "urn:server1",
                It.IsAny<ConfiguredEndpoint>(),
                It.IsAny<CancellationToken>()), Times.Once);
        }

        /// <summary>
        /// Verifies that transparent redundancy discovery reads the current server identifier.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoReadsTransparentCurrentServerIdAsync()
        {
            Mock<ISession> mockSession = CreateMockSession(
                redundancySupport: (int)RedundancySupport.Transparent,
                serviceLevel: ServiceLevels.HealthyMinimum,
                currentServerId: "server-a");

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);

            Assert.That(info.CurrentServerId, Is.EqualTo("server-a"));
        }

        /// <summary>
        /// Verifies that failed redundancy reads report no redundancy, an inaccessible service level, and no peers.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoHandlesReadErrorsAsync()
        {
            Mock<ISession> mockSession = CreateMockSessionWithBadStatus();

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);

            Assert.That(info.Mode, Is.EqualTo(RedundancySupport.None));
            Assert.That(info.ServiceLevel, Is.Zero);
            Assert.That(info.ServiceLevelAccessible, Is.False);
            Assert.That(info.RedundantServers, Is.Empty);
        }

        /// <summary>
        /// Verifies that malformed optional redundancy nodes preserve the discovered mode without inventing peers.
        /// </summary>
        [Test]
        public async Task FetchRedundancyInfoHandlesMalformedOptionalNodesAsync()
        {
            Mock<ISession> mockSession = CreateMockSessionWithThrowingOptionalReads();

            ServerRedundancyInfo info = await m_handler.FetchRedundancyInfoAsync(
                mockSession.Object).ConfigureAwait(false);

            Assert.That(info.Mode, Is.EqualTo(RedundancySupport.Hot));
            Assert.That(info.RedundantServers, Is.Empty);
        }

        /// <summary>
        /// Verifies that discovery and failover APIs reject null sessions, redundancy information, and endpoints.
        /// </summary>
        [Test]
        public void NullArgumentsThrow()
        {
            var info = new ServerRedundancyInfo();
            ConfiguredEndpoint endpoint = CreateCurrentEndpoint("urn:current");

            Assert.That(
                async () => await m_handler.FetchRedundancyInfoAsync(null!).ConfigureAwait(false),
                Throws.ArgumentNullException);
            Assert.That(() => m_handler.ShouldFailover(null!, endpoint), Throws.ArgumentNullException);
            Assert.That(() => m_handler.ShouldFailover(info, null!), Throws.ArgumentNullException);
            Assert.That(() => m_handler.SelectFailoverTarget(null!, endpoint), Throws.ArgumentNullException);
            Assert.That(() => m_handler.SelectFailoverTarget(info, null!), Throws.ArgumentNullException);
        }

        private static RedundantServer CreateServerInfo(
            string uri,
            byte serviceLevel,
            ServerState state)
        {
            return new RedundantServer
            {
                ServerUri = uri,
                ServiceLevel = serviceLevel,
                ServerState = state,
                Endpoint = CreateEndpoint(uri, $"opc.tcp://{uri[4..]}:4840")
            };
        }

        private static ConfiguredEndpoint CreateCurrentEndpoint(string applicationUri)
        {
            return CreateEndpoint(applicationUri, "opc.tcp://current:4840");
        }

        private static ConfiguredEndpoint CreateEndpoint(string applicationUri, string endpointUrl)
        {
            var description = new EndpointDescription
            {
                EndpointUrl = endpointUrl,
                SecurityMode = MessageSecurityMode.None,
                SecurityPolicyUri = SecurityPolicies.None,
                Server = new ApplicationDescription
                {
                    ApplicationUri = applicationUri,
                    DiscoveryUrls = new ArrayOf<string>(new[] { endpointUrl })
                }
            };

            return new ConfiguredEndpoint(null, description, configuration: null);
        }

        private static Mock<ISession> CreateMockSession(
            int redundancySupport,
            byte serviceLevel,
            RedundantServerDataType[]? redundantServers = null,
            string[]? serverUris = null,
            string currentServerId = "")
        {
            var mock = new Mock<ISession>();
            mock.SetupGet(s => s.ConfiguredEndpoint)
                .Returns(CreateCurrentEndpoint("urn:current"));

            mock.Setup(s => s.ReadAsync(
                    It.IsAny<RequestHeader>(),
                    It.IsAny<double>(),
                    It.IsAny<TimestampsToReturn>(),
                    It.Is<ArrayOf<ReadValueId>>(r => r.Count == 5),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(new ReadResponse
                {
                    Results =
                    [
                        new DataValue(new Variant(redundancySupport), StatusCodes.Good),
                        new DataValue(new Variant(serviceLevel), StatusCodes.Good),
                        redundantServers == null
                            ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
                            : CreateRedundantServerArrayValue(redundantServers),
                        serverUris == null
                            ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
                            : new DataValue(new Variant(new ArrayOf<string>(serverUris)), StatusCodes.Good),
                        new DataValue(new Variant(DateTime.MinValue), StatusCodes.Good)
                    ],
                    DiagnosticInfos = []
                });

            SetupSingleNodeRead(
                mock,
                VariableIds.Server_ServerRedundancy_CurrentServerId,
                string.IsNullOrEmpty(currentServerId)
                    ? new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
                    : new DataValue(new Variant(currentServerId), StatusCodes.Good));

            return mock;
        }

        private static void SetupSingleNodeRead(
            Mock<ISession> mock,
            NodeId nodeId,
            DataValue result)
        {
            mock.Setup(s => s.ReadAsync(
                    It.IsAny<RequestHeader>(),
                    It.IsAny<double>(),
                    It.IsAny<TimestampsToReturn>(),
                    It.Is<ArrayOf<ReadValueId>>(r => r.Count == 1 && r[0].NodeId == nodeId),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(new ReadResponse
                {
                    Results = [result],
                    DiagnosticInfos = []
                });
        }

        private static void VerifyBatchedRedundancyRead(Mock<ISession> mock)
        {
            mock.Verify(s => s.ReadAsync(
                It.IsAny<RequestHeader>(),
                It.IsAny<double>(),
                It.IsAny<TimestampsToReturn>(),
                It.Is<ArrayOf<ReadValueId>>(r => r.Count == 5),
                It.IsAny<CancellationToken>()), Times.Once);
            mock.Verify(s => s.ReadAsync(
                It.IsAny<RequestHeader>(),
                It.IsAny<double>(),
                It.IsAny<TimestampsToReturn>(),
                It.Is<ArrayOf<ReadValueId>>(r =>
                    r.Count == 1 &&
                    (r[0].NodeId == VariableIds.Server_ServerRedundancy_RedundantServerArray ||
                        r[0].NodeId == VariableIds.Server_ServerRedundancy_ServerUriArray)),
                It.IsAny<CancellationToken>()), Times.Never);
        }

        private static DataValue CreateRedundantServerArrayValue(
            RedundantServerDataType[] redundantServers)
        {
            ArrayOf<ExtensionObject> extensionObjects =
                Array.ConvertAll(redundantServers, s => new ExtensionObject(s));

            return new DataValue(new Variant(extensionObjects), StatusCodes.Good);
        }

        private static Mock<ISession> CreateMockSessionWithBadStatus()
        {
            var mock = new Mock<ISession>();

            mock.Setup(s => s.ReadAsync(
                    It.IsAny<RequestHeader>(),
                    It.IsAny<double>(),
                    It.IsAny<TimestampsToReturn>(),
                    It.Is<ArrayOf<ReadValueId>>(r => r.Count == 5),
                    It.IsAny<CancellationToken>()))
                .ReturnsAsync(new ReadResponse
                {
                    Results =
                    [
                        new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
                        new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
                        new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
                        new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown),
                        new DataValue(Variant.Null, StatusCodes.BadNodeIdUnknown)
                    ],
                    DiagnosticInfos = []
                });

            return mock;
        }

        private static Mock<ISession> CreateMockSessionWithThrowingOptionalReads()
        {
            return CreateMockSession(
                redundancySupport: (int)RedundancySupport.Hot,
                serviceLevel: ServiceLevels.NoData);
        }

        private sealed class FixedTimeProvider : TimeProvider
        {
            public FixedTimeProvider(DateTime utcNow)
            {
                m_utcNow = utcNow;
            }

            public override DateTimeOffset GetUtcNow()
            {
                return m_utcNow;
            }

            private readonly DateTimeOffset m_utcNow;
        }
    }
}