/* ======================================================================== * Copyright (c) 2005-2025 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Opc.Ua; using Opc.Ua.Redundancy; using Opc.Ua.Redundancy.Server; using Opc.Ua.Server; namespace RedundantServer { /// /// Factory that produces instances for DI-hosted servers. /// public sealed class HaSampleNodeManagerFactory : IAsyncNodeManagerFactory { private const string NamespaceUri = "http://opcfoundation.org/UA/Samples/HighAvailability"; private readonly ILeaderElection m_leaderElection; private readonly HaSampleReplicaInfo m_replicaInfo; private readonly IDistributedValueCache? m_valueCache; /// /// Creates a factory using the distributed leader-election service registered by the host. /// /// The leader-election service that identifies the active writer replica. /// The local replica identity published by sample variables. /// /// The distributed value cache registered by UseDistributedAddressSpace, when present. Injected as /// a sequence so active/active and single-instance topologies (which register no cache) resolve to empty. /// public HaSampleNodeManagerFactory( ILeaderElection leaderElection, HaSampleReplicaInfo replicaInfo, IEnumerable valueCaches) { m_leaderElection = leaderElection ?? throw new ArgumentNullException(nameof(leaderElection)); m_replicaInfo = replicaInfo ?? throw new ArgumentNullException(nameof(replicaInfo)); m_valueCache = valueCaches?.FirstOrDefault(); } /// public ArrayOf NamespacesUris => [NamespaceUri]; /// public ValueTask CreateAsync( IServerInternal server, ApplicationConfiguration configuration, CancellationToken cancellationToken = default) { _ = configuration; _ = cancellationToken; #pragma warning disable CA2000 // ownership transfers to the server var manager = new HaSampleNodeManager( server, m_leaderElection, m_replicaInfo, m_valueCache, [.. NamespacesUris]); #pragma warning restore CA2000 return new ValueTask(manager); } } /// /// Carries the local replica identity into sample node managers created by dependency injection. /// public sealed class HaSampleReplicaInfo { /// /// Creates a replica identity descriptor. /// /// The unique local high-availability node id. public HaSampleReplicaInfo(string nodeId) { NodeId = nodeId; } /// /// Gets the unique local high-availability node id. /// public string NodeId { get; } } /// /// Minimal address space that participates in distributed replication. /// public sealed class HaSampleNodeManager : AsyncCustomNodeManager { private readonly ILeaderElection m_leaderElection; private readonly HaSampleReplicaInfo m_replicaInfo; private readonly IDistributedValueCache? m_valueCache; private readonly CancellationTokenSource m_simulationCts = new(); private readonly Lock m_updateLock = new(); private static readonly TimeSpan s_valueFreshness = TimeSpan.FromSeconds(10); private BaseDataVariableState? m_counter; private BaseDataVariableState? m_activeReplica; private Task? m_simulationTask; private int m_counterValue; /// /// Creates the high-availability sample node manager. /// /// The server that owns the node manager. /// The leader-election service that gates sample writes. /// The local replica identity published by sample variables. /// /// The distributed value cache used to share the Counter value across the replica set, or null when /// the topology registers none (active/active and single-instance). /// /// The namespace URIs exposed by this node manager. public HaSampleNodeManager( IServerInternal server, ILeaderElection leaderElection, HaSampleReplicaInfo replicaInfo, IDistributedValueCache? valueCache, params string[] namespaceUris) : base(server, namespaceUris) { m_leaderElection = leaderElection ?? throw new ArgumentNullException(nameof(leaderElection)); m_replicaInfo = replicaInfo ?? throw new ArgumentNullException(nameof(replicaInfo)); m_valueCache = valueCache; } /// public override async ValueTask CreateAddressSpaceAsync( IDictionary> externalReferences, CancellationToken cancellationToken = default) { if (!externalReferences.TryGetValue(ObjectIds.ObjectsFolder, out IList? references)) { externalReferences[ObjectIds.ObjectsFolder] = references = []; } ushort namespaceIndex = NamespaceIndexes[0]; FolderState folder = CreateFolder(null, namespaceIndex, "HighAvailability", "High Availability"); folder.AddReference(ReferenceTypeIds.Organizes, true, ObjectIds.ObjectsFolder); references.Add(new NodeStateReference(ReferenceTypeIds.Organizes, false, folder.NodeId)); m_counter = CreateVariable(folder, namespaceIndex, "Counter", DataTypeIds.Int32, Variant.From(0)); if (m_valueCache != null) { // Opt the Counter's read/write callbacks into the distributed // value cache (extension beyond OPC 10000-4 ยง6.6): the active // replica caches each new value write-through, and every replica // serves the last value shared through the store while it is // fresh (falling back to the local value). This is how the // Counter is shared across the set and continues after failover; // monitored items keep reading through the normal pipeline. m_counter.EnableDistributedValueParticipation( m_valueCache, s_valueFreshness, _ => new ValueTask(ReadLocalCounter())); } m_activeReplica = CreateVariable( folder, namespaceIndex, "ActiveReplica", DataTypeIds.String, Variant.From("unknown")); await AddPredefinedNodeAsync(SystemContext, folder, cancellationToken).ConfigureAwait(false); StartSimulation(); } /// protected override void Dispose(bool disposing) { if (disposing) { m_simulationCts.Cancel(); m_simulationCts.Dispose(); } base.Dispose(disposing); } private static FolderState CreateFolder(NodeState? parent, ushort namespaceIndex, string path, string name) { var folder = new FolderState(parent) { SymbolicName = name, ReferenceTypeId = ReferenceTypeIds.Organizes, TypeDefinitionId = ObjectTypeIds.FolderType, NodeId = new NodeId(path, namespaceIndex), BrowseName = new QualifiedName(path, namespaceIndex), DisplayName = new LocalizedText("en", name), WriteMask = AttributeWriteMask.None, UserWriteMask = AttributeWriteMask.None, EventNotifier = EventNotifiers.None }; parent?.AddChild(folder); return folder; } private static BaseDataVariableState CreateVariable( NodeState parent, ushort namespaceIndex, string name, NodeId dataType, Variant initialValue) { var variable = new BaseDataVariableState(parent) { SymbolicName = name, ReferenceTypeId = ReferenceTypeIds.Organizes, TypeDefinitionId = VariableTypeIds.BaseDataVariableType, NodeId = new NodeId(name, namespaceIndex), BrowseName = new QualifiedName(name, namespaceIndex), DisplayName = new LocalizedText("en", name), WriteMask = AttributeWriteMask.None, UserWriteMask = AttributeWriteMask.None, DataType = dataType, ValueRank = ValueRanks.Scalar, AccessLevel = AccessLevels.CurrentReadOrWrite, UserAccessLevel = AccessLevels.CurrentReadOrWrite, Historizing = false, Value = initialValue, StatusCode = StatusCodes.Good, Timestamp = DateTime.UtcNow }; parent.AddChild(variable); return variable; } private void StartSimulation() { m_simulationTask ??= Task.Run(() => RunSimulationAsync(m_simulationCts.Token)); } private async Task RunSimulationAsync(CancellationToken cancellationToken) { using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1)); bool wasLeader = false; DateTime lastHeartbeat = DateTime.MinValue; while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)) { if (!m_leaderElection.IsLeader) { if (wasLeader) { // A failover moved the write role away from this replica. m_logger.ReplicaBecameStandby(m_replicaInfo.NodeId); } wasLeader = false; UpdateActiveReplica("standby"); continue; } if (!wasLeader) { // This replica has just become the active writer (initial // election or a failover). Continue the Counter from the // last value shared through the distributed store instead of // restarting from the local count. await SeedCounterFromCacheAsync(cancellationToken).ConfigureAwait(false); if (m_logger.IsEnabled(LogLevel.Information)) { m_logger.ReplicaBecameActiveWriter( m_replicaInfo.NodeId, Volatile.Read(ref m_counterValue)); } wasLeader = true; } await UpdateCounterAsync(cancellationToken).ConfigureAwait(false); UpdateActiveReplica(m_replicaInfo.NodeId); // Periodic heartbeat so the log shows liveness, which replica is // producing which Counter values, and (across replicas) the // per-replica divergence a client observes on failover. DateTime now = DateTime.UtcNow; if (now - lastHeartbeat >= TimeSpan.FromSeconds(5)) { lastHeartbeat = now; if (m_logger.IsEnabled(LogLevel.Information)) { m_logger.ReplicaActive( m_replicaInfo.NodeId, Volatile.Read(ref m_counterValue)); } } } } private async Task UpdateCounterAsync(CancellationToken cancellationToken) { BaseDataVariableState? counter = m_counter; if (counter == null) { return; } int value = Interlocked.Increment(ref m_counterValue); lock (m_updateLock) { counter.Value = value; counter.Timestamp = DateTime.UtcNow; counter.ClearChangeMasks(SystemContext, false); } // Share the new value through the distributed store so standby // replicas can serve it and a promoted replica continues from it. if (m_valueCache != null) { try { await m_valueCache .CacheAsync( counter.NodeId, new DataValue(Variant.From(value), StatusCodes.Good, DateTimeUtc.Now), cancellationToken) .ConfigureAwait(false); } catch (InvalidOperationException) { // The distributed store is not ready yet (the server is // still starting); the value is shared on the next tick. } } } private async Task SeedCounterFromCacheAsync(CancellationToken cancellationToken) { BaseDataVariableState? counter = m_counter; if (m_valueCache == null || counter == null) { return; } try { (bool _, DataValue cached) = await m_valueCache .TryGetAsync(counter.NodeId, s_valueFreshness, cancellationToken) .ConfigureAwait(false); if (cached.WrappedValue.TryGetValue(out int last) && last > Volatile.Read(ref m_counterValue)) { Interlocked.Exchange(ref m_counterValue, last); lock (m_updateLock) { counter.Value = last; counter.Timestamp = DateTime.UtcNow; counter.ClearChangeMasks(SystemContext, false); } } } catch (InvalidOperationException) { // The distributed store is not ready yet; start from the local // value and share on the next tick. } } private DataValue ReadLocalCounter() { return new DataValue( Variant.From(Volatile.Read(ref m_counterValue)), StatusCodes.Good, DateTimeUtc.Now); } private void UpdateActiveReplica(string value) { BaseDataVariableState? activeReplica = m_activeReplica; if (activeReplica == null) { return; } lock (m_updateLock) { activeReplica.Value = value; activeReplica.Timestamp = DateTime.UtcNow; activeReplica.ClearChangeMasks(SystemContext, false); } } } internal static partial class HaSampleNodeManagerLog { [LoggerMessage(EventId = RedundantServerEventIds.HaSampleNodeManager + 0, Level = LogLevel.Information, Message = "HA: replica {ReplicaId} became STANDBY (no longer the active writer).")] public static partial void ReplicaBecameStandby(this ILogger logger, string replicaId); [LoggerMessage(EventId = RedundantServerEventIds.HaSampleNodeManager + 1, Level = LogLevel.Information, Message = "HA: replica {ReplicaId} became ACTIVE writer (Counter={Counter}).")] public static partial void ReplicaBecameActiveWriter(this ILogger logger, string replicaId, int counter); [LoggerMessage(EventId = RedundantServerEventIds.HaSampleNodeManager + 2, Level = LogLevel.Information, Message = "HA: replica {ReplicaId} ACTIVE, Counter={Counter}.")] public static partial void ReplicaActive(this ILogger logger, string replicaId, int counter); } }