From 029ac0f33f68a982fcb6c1c72e538ac49a8f1531 Mon Sep 17 00:00:00 2001 From: Klaus Landsdorf Date: Thu, 1 Oct 2026 17:06:06 +0200 Subject: [PATCH] Kubernetes readiness: create the listener on Start and move the default probe port off 8080 (breaking default) (#4613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Two commits that can be taken separately. 1. **`fix(kubernetes): create the readiness listener only when started`** (not breaking). The managed HttpListener (Linux, macOS) binds its endpoint when a listener that has prefixes is closed. Because `KubernetesReadinessServer` created the listener in its constructor, disposing a server that was never started bound the probe port and threw "address already in use" if anything else held it. The listener is now created in `Start`, and `Start` after `DisposeAsync` throws `ObjectDisposedException` explicitly. With port 8080 occupied, master fails 4 of 99 tests in `Opc.Ua.Redundancy.Kubernetes.Tests`; with this commit all pass. New test: `DisposingAnUnstartedServerDoesNotBindItsPort`. 2. **`fix(kubernetes)!: move the readiness probe off the web-application port 8080`** โ€” **breaking (default only):** `KubernetesReadinessOptions.Port` now defaults to the new `KubernetesReadinessOptions.DefaultPort` = 4852 instead of 8080. - 8080 is the de-facto default of web applications and reverse proxies, and the probe shares the pod (with hostNetwork, the node) with the application and its sidecars. - 4852 sits next to the OPC UA ports 4840/4843 and is unassigned in the IANA registry (4852โ€“4866, checked 2026-09-30). One de-facto use we found: Beckhoff IPCs run an "IPC Diagnostics" OPC UA server on 4852 on Windows images; that only matters with hostNetwork on such a node. If you prefer another port from the unassigned range, commit 1 stands on its own. - Migration: set the manifest `containerPort` to 4852, or keep the old port with `UseKubernetesReadiness(o => o.Port = 8080)`. `docs/Kubernetes.md`, `NugetREADME.md` and a new `docs/MigrationGuide.md` section are updated. - The package has shipped only in 2.0.0 previews, so changing the default before GA avoids locking the collision in. Tests: `Opc.Ua.Redundancy.Kubernetes.Tests` 100/100 on net10.0 and net8.0. Fork CI (full matrix incl. macOS) and CodeQL: green โ€” [CI run 36769864949](https://github.com/biancode/UA-.NETStandard/actions/runs/36769864949) (full matrix incl. macOS), [CodeQL run 36769869626](https://github.com/biancode/UA-.NETStandard/actions/runs/36769869626). ## Related Issues - Fixes #4612 ## Checklist - [x] I have signed the [CLA](https://opcfoundation.org/license/cla/ContributorLicenseAgreementv1.0.pdf) and read the [CONTRIBUTING](https://github.com/OPCFoundation/UA-.NETStandard/blob/master/CONTRIBUTING.md) doc. - [x] I have added tests that prove my fix is effective or that my feature works and increased code coverage. - [x] I have added all necessary documentation. - [x] I have verified that my changes do not introduce (new) build or analyzer warnings. - [ ] I ran **all** tests locally using the **UA.slnx** solution against at least .net **framework** and .net **10**, and all passed. (Affected suites on net10.0 and net8.0 locally on macOS; .NET Framework is covered by the fork CI's Windows net48 legs.) - [ ] I fixed **all** failing and flaky tests in the CI pipelines and **all** CodeQL warnings. - [ ] I have addressed **all** PR feedback received. --- docs/Kubernetes.md | 6 +-- .../Health/KubernetesReadinessOptions.cs | 16 ++++++- .../Health/KubernetesReadinessServer.cs | 41 +++++++++++----- .../NugetREADME.md | 4 +- .../KubernetesReadinessServerHttpTests.cs | 47 +++++++++++++++++++ 5 files changed, 96 insertions(+), 18 deletions(-) diff --git a/docs/Kubernetes.md b/docs/Kubernetes.md index 857f46046e..ac6b22fb97 100644 --- a/docs/Kubernetes.md +++ b/docs/Kubernetes.md @@ -116,7 +116,7 @@ services.AddOpcUa() }) .UseKubernetesReadiness(options => { - options.Port = 8080; + options.Port = KubernetesReadinessOptions.DefaultPort; // 4852 options.ReadinessPath = "/readyz"; options.LivenessPath = "/livez"; options.ReadyMinimumServiceLevel = ServiceLevels.HealthyMinimum; @@ -278,7 +278,7 @@ spec: - name: opcua-tcp containerPort: 4840 - name: health - containerPort: 8080 + containerPort: 4852 readinessProbe: httpGet: path: /readyz @@ -297,7 +297,7 @@ A Deployment variant is nearly identical: replace `kind: StatefulSet` with `kind ## Readiness and ServiceLevel -`UseKubernetesReadiness` starts a small HTTP listener. `/livez` returns success while the process can answer the probe. `/readyz` returns HTTP 200 only when `IServiceLevelProvider.GetServiceLevel()` is at least `ReadyMinimumServiceLevel`, which defaults to the Healthy sub-range (`200`). This lets a client-facing Service route only to the current leader or otherwise healthy replicas. +`UseKubernetesReadiness` starts a small HTTP listener on port 4852 (`KubernetesReadinessOptions.DefaultPort`) when the server has started. The default deliberately avoids the ports web applications commonly use, such as 8080 or 9090: the probe shares the pod (and, with host networking, the node) with the application and its sidecars, so such a default collides with them. 4852 sits next to the OPC UA ports 4840/4843 and is unassigned in the IANA service name and port number registry; set `Port` if it is taken in your environment. `/livez` returns success while the process can answer the probe. `/readyz` returns HTTP 200 only when `IServiceLevelProvider.GetServiceLevel()` is at least `ReadyMinimumServiceLevel`, which defaults to the Healthy sub-range (`200`). This lets a client-facing Service route only to the current leader or otherwise healthy replicas. For Warm active/passive deployments, `LeaderServiceLevelProvider` normally makes the leader Healthy and standbys Degraded; readiness therefore routes traffic to the leader. For Hot load-balanced deployments, use the Healthy 200-255 sub-range to reflect load and keep all healthy pods ready. diff --git a/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessOptions.cs b/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessOptions.cs index a1b68199cb..131caae42b 100644 --- a/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessOptions.cs +++ b/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessOptions.cs @@ -40,9 +40,9 @@ namespace Opc.Ua.Redundancy.Kubernetes public string Host { get; set; } = "+"; /// - /// Gets or sets the listener port. + /// Gets or sets the listener port. The default is . /// - public int Port { get; set; } = 8080; + public int Port { get; set; } = DefaultPort; /// /// Gets or sets the readiness path. @@ -58,5 +58,17 @@ namespace Opc.Ua.Redundancy.Kubernetes /// Gets or sets the minimum OPC 10000-4 ยง6.6.2.4.2 ServiceLevel considered ready. /// public byte ReadyMinimumServiceLevel { get; set; } = 200; + + /// + /// The default listener port, 4852. + /// + /// + /// The probe shares the pod (and, with host networking, the node) with the + /// application and its sidecars, so the default stays off the ports web + /// applications and reverse proxies commonly use (8080, 8000, 9090, ...). + /// 4852 lies next to the OPC UA ports 4840/4843 and is unassigned in the + /// IANA service name and port number registry (4852-4866). + /// + public const int DefaultPort = 4852; } } diff --git a/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessServer.cs b/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessServer.cs index a99804ce1b..7b01c2eeb2 100644 --- a/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessServer.cs +++ b/src/Opc.Ua.Redundancy.Kubernetes/Health/KubernetesReadinessServer.cs @@ -59,31 +59,47 @@ namespace Opc.Ua.Redundancy.Kubernetes ?? throw new ArgumentNullException(nameof(serviceLevelProvider)); m_options = options ?? throw new ArgumentNullException(nameof(options)); m_logger = logger; - m_listener = new HttpListener(); - m_listener.Prefixes.Add(ToPrefix(m_options.Host, m_options.Port, m_options.ReadinessPath)); - m_listener.Prefixes.Add(ToPrefix(m_options.Host, m_options.Port, m_options.LivenessPath)); + m_prefixes = + [ + ToPrefix(m_options.Host, m_options.Port, m_options.ReadinessPath), + ToPrefix(m_options.Host, m_options.Port, m_options.LivenessPath) + ]; } /// /// Starts the HTTP listener. /// + /// + /// The listener is created here rather than in the constructor: the managed + /// binds its port when a listener that has prefixes + /// is closed, so a server that was built but never started would otherwise + /// bind - and fail with "address already in use" - on disposal. + /// + /// The server was disposed. public void Start() { lock (m_lock) { - if (m_started) + ObjectDisposedException.ThrowIf(m_disposed, this); + if (m_listener != null) { return; } - m_started = true; - m_listener.Start(); - m_loop = Task.Run(() => ListenAsync(m_cts.Token)); + var listener = new HttpListener(); + foreach (string prefix in m_prefixes) + { + listener.Prefixes.Add(prefix); + } + listener.Start(); + m_listener = listener; + m_loop = Task.Run(() => ListenAsync(listener, m_cts.Token)); } } /// public async ValueTask DisposeAsync() { + HttpListener? listener; lock (m_lock) { if (m_disposed) @@ -91,10 +107,11 @@ namespace Opc.Ua.Redundancy.Kubernetes return; } m_disposed = true; + listener = m_listener; } m_cts.Cancel(); - m_listener.Close(); + listener?.Close(); if (m_loop != null) { try @@ -133,14 +150,14 @@ namespace Opc.Ua.Redundancy.Kubernetes return serviceLevel >= readyMinimumServiceLevel; } - private async Task ListenAsync(CancellationToken ct) + private async Task ListenAsync(HttpListener listener, CancellationToken ct) { while (!ct.IsCancellationRequested) { HttpListenerContext context; try { - context = await m_listener.GetContextAsync().ConfigureAwait(false); + context = await listener.GetContextAsync().ConfigureAwait(false); } catch (ObjectDisposedException) { @@ -244,14 +261,14 @@ namespace Opc.Ua.Redundancy.Kubernetes private readonly IServiceLevelProvider m_serviceLevelProvider; private readonly KubernetesReadinessOptions m_options; private readonly ILogger? m_logger; - private readonly HttpListener m_listener; + private readonly string[] m_prefixes; private readonly Lock m_lock = new(); private readonly CancellationTokenSource m_cts = new(); private readonly SemaphoreSlim m_handlerSlots = new(kMaxConcurrentRequests, kMaxConcurrentRequests); private const int kMaxConcurrentRequests = 16; private static readonly TimeSpan kDrainTimeout = TimeSpan.FromSeconds(5); + private HttpListener? m_listener; private Task? m_loop; - private bool m_started; private bool m_disposed; } diff --git a/src/Opc.Ua.Redundancy.Kubernetes/NugetREADME.md b/src/Opc.Ua.Redundancy.Kubernetes/NugetREADME.md index 889132e526..e3d17d90da 100644 --- a/src/Opc.Ua.Redundancy.Kubernetes/NugetREADME.md +++ b/src/Opc.Ua.Redundancy.Kubernetes/NugetREADME.md @@ -14,9 +14,11 @@ services.AddOpcUa() .UseDistributedAddressSpace(options => options.UseLeaderElection = true) .UseKubernetesLeaderElection(options => options.LeaseName = "opcua-ha") .UseKubernetesPeerDiscovery(options => options.ServiceName = "opcua-ha-headless") - .UseKubernetesReadiness(options => options.Port = 8080); + .UseKubernetesReadiness(); // /readyz and /livez on port 4852 ``` +The readiness probe listens on `KubernetesReadinessOptions.DefaultPort` (4852) unless `Port` is set. The default stays off the ports web applications commonly use (8080, 9090, ...), which would collide with the application or a sidecar in the same pod. + For strong consistency, wire a Raft cluster from the StatefulSet ordinals and compose it with `UseRedundancyConsistency`: ```csharp diff --git a/tests/Opc.Ua.Redundancy.Kubernetes.Tests/Health/KubernetesReadinessServerHttpTests.cs b/tests/Opc.Ua.Redundancy.Kubernetes.Tests/Health/KubernetesReadinessServerHttpTests.cs index a3d2df0237..d238b5ab13 100644 --- a/tests/Opc.Ua.Redundancy.Kubernetes.Tests/Health/KubernetesReadinessServerHttpTests.cs +++ b/tests/Opc.Ua.Redundancy.Kubernetes.Tests/Health/KubernetesReadinessServerHttpTests.cs @@ -128,6 +128,53 @@ namespace Opc.Ua.Redundancy.Kubernetes.Tests Assert.That(notReadyServer.IsReady(), Is.False); } + [Test] + public void DisposingAnUnstartedServerDoesNotBindItsPort() + { + // Another process holds the port on the wildcard address, where the + // default "+" host binds. A server that was never started must not + // try to bind the port when it is disposed. + var occupant = new TcpListener(Socket.OSSupportsIPv6 ? IPAddress.IPv6Any : IPAddress.Any, 0); + if (Socket.OSSupportsIPv6) + { + occupant.Server.DualMode = true; + } + occupant.Start(); + try + { + int port = ((IPEndPoint)occupant.LocalEndpoint).Port; + var server = new KubernetesReadinessServer( + new ConstantServiceLevelProvider(255), + new KubernetesReadinessOptions { Port = port }); + + Assert.That(async () => await server.DisposeAsync().ConfigureAwait(false), Throws.Nothing); + } + finally + { + occupant.Stop(); + } + } + + [Test] + public async Task StartAfterDisposeIsRejectedAsync() + { + var server = new KubernetesReadinessServer( + new ConstantServiceLevelProvider(255), + NewOptions(GetFreePort())); + await server.DisposeAsync().ConfigureAwait(false); + + Assert.That(server.Start, Throws.TypeOf()); + } + + [Test] + public void DefaultPortStaysOffCommonWebApplicationPorts() + { + var options = new KubernetesReadinessOptions(); + + Assert.That(options.Port, Is.EqualTo(KubernetesReadinessOptions.DefaultPort)); + Assert.That(options.Port, Is.EqualTo(4852)); + } + [Test] public void ConstructorRejectsNullServiceLevelProvider() {