diff --git a/docs/ResourceIsolation.md b/docs/ResourceIsolation.md index 03b5538302..7339b87046 100644 --- a/docs/ResourceIsolation.md +++ b/docs/ResourceIsolation.md @@ -593,7 +593,9 @@ bootstrap, reconnect, control and trusted floors, so the shared capacity equals the configured value. For example, with `MaxQueuedRequestCount = 10000`, ten clients can each keep 1,000 requests outstanding. Balanced then sizes the RequestQueue stage at 10,003 entries. RequestExecution keeps its floors inside -the worker limit, so an idle worker remains available for protected requests. +the worker limit, so an idle worker remains available for protected requests: +its total is `MaxRequestThreadCount`, and startup fails when that leaves no +worker for ordinary requests (Balanced needs at least 4). An explicit `Stages[].Capacity` is still an absolute total that includes the floors, and it cannot exceed the default total. diff --git a/perf/ServerLoadHarness/NativeWorkerBenchmark.cs b/perf/ServerLoadHarness/NativeWorkerBenchmark.cs index 644d68a695..ed408ba557 100644 --- a/perf/ServerLoadHarness/NativeWorkerBenchmark.cs +++ b/perf/ServerLoadHarness/NativeWorkerBenchmark.cs @@ -413,7 +413,9 @@ namespace Opc.Ua.Perf.ServerLoadHarness FormattableString.Invariant( $"--counters {(counters ? Program.Arg("nodes", "1") : "0")} --sampling {Program.Arg("sampling", "10")} ") + FormattableString.Invariant( - $"--counter-lock {Program.Arg("counter-lock", "1")} --async-node-manager {Program.Arg("async-node-manager", "1")}"), + $"--counter-lock {Program.Arg("counter-lock", "1")} --async-node-manager {Program.Arg("async-node-manager", "1")} ") + + FormattableString.Invariant( + $"--isolation {Program.Arg("isolation", "default")}"), // the benchmark's .NET workers run with Server GC; SERVER_DOTNET_gcServer=0 overrides it new Dictionary(StringComparer.Ordinal) { ["DOTNET_gcServer"] = "1" }); string ready = handle.m_lines.Take(); diff --git a/src/Opc.Ua.Core/Stack/Server/ServerBase.RequestQueue.cs b/src/Opc.Ua.Core/Stack/Server/ServerBase.RequestQueue.cs index ce432c36a3..e75407211f 100644 --- a/src/Opc.Ua.Core/Stack/Server/ServerBase.RequestQueue.cs +++ b/src/Opc.Ua.Core/Stack/Server/ServerBase.RequestQueue.cs @@ -124,7 +124,7 @@ namespace Opc.Ua description: "Requests executing and holding a permit or worker."); m_meter.CreateObservableGauge( "opcua.server.request_queue.parked", - () => Math.Max(0, Volatile.Read(ref m_processingCount) - Volatile.Read(ref m_activeThreadCount)), + () => Volatile.Read(ref m_parkedCount), description: "Requests parked (for example held Publish requests) after releasing their permit or worker."); m_workers = []; @@ -330,6 +330,7 @@ namespace Opc.Ua if (!processing.IsCompleted) { await Task.WhenAny(processing, parkSink.ParkedTask).ConfigureAwait(false); + CountWhileParked(processing); } } else @@ -553,6 +554,7 @@ namespace Opc.Ua if (!processing.IsCompleted) { await Task.WhenAny(processing, parkSink.ParkedTask).ConfigureAwait(false); + CountWhileParked(processing); } } finally @@ -566,6 +568,26 @@ namespace Opc.Ua // required here because ProcessRequestSafeAsync observes all faults. } + /// + /// Counts a request that released its permit or worker at the park point as + /// parked until its handler finishes. + /// + private void CountWhileParked(Task processing) + { + if (processing.IsCompleted) + { + return; + } + Interlocked.Increment(ref m_parkedCount); + // ProcessRequestSafeAsync never throws, so the continuation only counts. + _ = processing.ContinueWith( + static (_, state) => Interlocked.Decrement(ref ((RequestQueue)state!).m_parkedCount), + this, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously, + TaskScheduler.Default); + } + /// /// Invokes the server request handler and traps any unexpected error the same /// way the legacy inline path did, guaranteeing the request is always faulted @@ -793,6 +815,12 @@ namespace Opc.Ua /// private int m_processingCount; + /// + /// Number of handlers that parked and released their permit or worker, + /// changed with . + /// + private int m_parkedCount; + /// /// Number of requests executing rather than parked or waiting: the permits in /// use, changed under , or the busy fair-queue diff --git a/src/Opc.Ua.Server/ResourceIsolation/ServerResourceIsolationOptions.cs b/src/Opc.Ua.Server/ResourceIsolation/ServerResourceIsolationOptions.cs index c0ec83ab39..1c961b1845 100644 --- a/src/Opc.Ua.Server/ResourceIsolation/ServerResourceIsolationOptions.cs +++ b/src/Opc.Ua.Server/ResourceIsolation/ServerResourceIsolationOptions.cs @@ -281,7 +281,9 @@ namespace Opc.Ua.Server ? rateLimits.MaxConcurrentSessionEstablishment : ServerRateLimitOptions.DefaultMaxConcurrentSessionEstablishment; int queuedCapacity = Math.Max(100, server.MaxQueuedRequestCount); - int executionCapacity = Math.Max(100, Math.Max(server.MinRequestThreadCount, server.MaxRequestThreadCount)); + // the request queue starts at most this many workers, so the execution stage cannot + // promise protected classes more than the workers can run + int executionCapacity = Math.Max(1, Math.Max(server.MinRequestThreadCount, server.MaxRequestThreadCount)); long retainedRequestSlots = 2L * queuedCapacity + executionCapacity; if (retainedRequestSlots > long.MaxValue / quotas.MaxMessageSize) { @@ -382,6 +384,13 @@ namespace Opc.Ua.Server SubtractFloor(ref remaining, owner.Floors[ii]); trustedTotal += owner.Floors[ii]; } + if (remaining < unit && ii == (int)ResourceIsolationStage.RequestExecution && item?.Capacity == null) + { + throw new ArgumentException( + $"MaxRequestThreadCount ({executionCapacity}) must exceed the requests reserved for " + + "bootstrap, reconnect, control and trusted owners. Increase MaxRequestThreadCount or " + + "reduce the RequestExecution reservations."); + } if (remaining < unit || hardLimit < Math.Max(Math.Max(bootstrapFloor, reconnectFloor), controlFloor)) { throw new ArgumentException($"The {(ResourceIsolationStage)ii} pools cannot hold an operation."); diff --git a/tests/Opc.Ua.Server.Tests/ResourceIsolation/IsolationConfigurationValidationTests.cs b/tests/Opc.Ua.Server.Tests/ResourceIsolation/IsolationConfigurationValidationTests.cs index 0c81d7a600..114527f077 100644 --- a/tests/Opc.Ua.Server.Tests/ResourceIsolation/IsolationConfigurationValidationTests.cs +++ b/tests/Opc.Ua.Server.Tests/ResourceIsolation/IsolationConfigurationValidationTests.cs @@ -60,6 +60,39 @@ namespace Opc.Ua.Server.Tests .And.Message.Contains("two minutes")); } + /// + /// The execution stage holds no more requests than the request queue has workers, + /// so reserved classes cannot be crowded out of the workers by ordinary requests. + /// + [Test] + public void RuntimePlanExecutionCapacityIsMaxRequestThreadCount() + { + ApplicationConfiguration configuration = Configuration(); + configuration.ServerConfiguration.MinRequestThreadCount = 1; + configuration.ServerConfiguration.MaxRequestThreadCount = 4; + + ResourceIsolationStagePlan execution = Options() + .CreateRuntimePlan(configuration, new ServerRateLimitOptions()) + .GetStage(ResourceIsolationStage.RequestExecution); + + Assert.That(execution.Capacity, Is.EqualTo(4)); + Assert.That(execution.SharedCapacity, Is.EqualTo(1)); + } + + /// + /// Too few workers for the reserved requests are named at startup. + /// + [Test] + public void RuntimePlanRejectsMaxRequestThreadCountBelowReservations() + { + ApplicationConfiguration configuration = Configuration(); + configuration.ServerConfiguration.MinRequestThreadCount = 1; + configuration.ServerConfiguration.MaxRequestThreadCount = 3; + + Assert.That(() => Options().CreateRuntimePlan(configuration, new ServerRateLimitOptions()), + Throws.ArgumentException.With.Message.Contains("MaxRequestThreadCount (3)")); + } + [Test] public async Task StartupRejectsInvalidTimeoutWithoutInstallingOrDisposingProvidersAsync( [Values] ServerResourceIsolationMode mode,