/* ======================================================================== * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ #if NET10_0 using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; namespace Opc.Ua.Tools.Tests { /// /// Contract tests for .github/scripts/update-test-durations.ps1, which /// rewrites the weights get-ci-matrix.ps1 packs test batches with. A /// regression there would silently skew every batch of every later run, /// so the selection rules are pinned against temporary summaries. /// [TestFixture] [NonParallelizable] public sealed class CiTestDurationsScriptTests { private string m_workDirectory = string.Empty; private string m_tablePath = string.Empty; private string m_resultsPath = string.Empty; [SetUp] public async Task SetUpAsync() { m_workDirectory = Path.Combine(Path.GetTempPath(), $"ci-durations-{Guid.NewGuid():N}"); m_resultsPath = Path.Combine(m_workDirectory, "results"); Directory.CreateDirectory(m_resultsPath); m_tablePath = Path.Combine(m_workDirectory, "ci-test-durations.json"); await File.WriteAllTextAsync( m_tablePath, """ { "source": "seed", "defaultMinutes": 5, "projects": { "Opc.Ua.Kept.Tests": 9, "Opc.Ua.Slow.Tests": 2 } } """).ConfigureAwait(false); } [TearDown] public void TearDown() { if (Directory.Exists(m_workDirectory)) { Directory.Delete(m_workDirectory, recursive: true); } } /// /// A project's weight is its slowest profile's passed test time, rounded /// up to whole minutes, plus one minute of incremental build. /// [Test] public async Task WeightIsTheSlowestPassedProfilePlusOneMinuteAsync() { await WriteSummaryAsync("linux", ("tests/Opc.Ua.Slow.Tests/Opc.Ua.Slow.Tests.csproj", "passed", 130)) .ConfigureAwait(false); await WriteSummaryAsync("macos", ("tests/Opc.Ua.Slow.Tests/Opc.Ua.Slow.Tests.csproj", "passed", 250)) .ConfigureAwait(false); ScriptResult result = await RunScriptAsync().ConfigureAwait(false); Assert.That(result.ExitCode, Is.Zero, result.Output); Assert.That(await ReadWeightsAsync().ConfigureAwait(false), Does.ContainKey("Opc.Ua.Slow.Tests").WithValue(6)); } /// /// Failed and not-applicable records say nothing about a healthy run's /// duration and must not move a weight, however long they took. /// [Test] public async Task FailedAndNotApplicableRecordsAreIgnoredAsync() { await WriteSummaryAsync( "windows", ("tests/Opc.Ua.Slow.Tests/Opc.Ua.Slow.Tests.csproj", "passed", 60), ("tests/Opc.Ua.Slow.Tests/Opc.Ua.Slow.Tests.csproj", "failed", 9999), ("tests/Opc.Ua.Kept.Tests/Opc.Ua.Kept.Tests.csproj", "not-applicable", 9999)).ConfigureAwait(false); ScriptResult result = await RunScriptAsync().ConfigureAwait(false); Assert.That(result.ExitCode, Is.Zero, result.Output); Dictionary weights = await ReadWeightsAsync().ConfigureAwait(false); Assert.That(weights, Does.ContainKey("Opc.Ua.Slow.Tests").WithValue(2)); Assert.That(weights, Does.ContainKey("Opc.Ua.Kept.Tests").WithValue(9)); } /// /// A project the run did not execute keeps its weight, a newly seen one /// is added, and the table keeps its default and records its source. /// [Test] public async Task UnobservedProjectsKeepTheirWeightAndNewOnesAreAddedAsync() { await WriteSummaryAsync("linux", ("tests/Opc.Ua.New.Tests/Opc.Ua.New.Tests.csproj", "passed", 30)) .ConfigureAwait(false); ScriptResult result = await RunScriptAsync().ConfigureAwait(false); Assert.That(result.ExitCode, Is.Zero, result.Output); Dictionary weights = await ReadWeightsAsync().ConfigureAwait(false); Assert.That(weights, Does.ContainKey("Opc.Ua.Kept.Tests").WithValue(9)); Assert.That(weights, Does.ContainKey("Opc.Ua.Slow.Tests").WithValue(2)); Assert.That(weights, Does.ContainKey("Opc.Ua.New.Tests").WithValue(2)); using JsonDocument document = JsonDocument.Parse(await File.ReadAllTextAsync(m_tablePath).ConfigureAwait(false)); Assert.That(document.RootElement.GetProperty("defaultMinutes").GetInt32(), Is.EqualTo(5)); Assert.That(document.RootElement.GetProperty("source").GetString(), Is.EqualTo("test run")); } /// /// Pointing the script at a directory without summaries is a mistake /// (usually a missing download), not a run that observed nothing. /// [Test] public async Task MissingSummariesFailWithoutTouchingTheTableAsync() { string before = await File.ReadAllTextAsync(m_tablePath).ConfigureAwait(false); ScriptResult result = await RunScriptAsync().ConfigureAwait(false); Assert.That(result.ExitCode, Is.Not.Zero); Assert.That(result.Output, Does.Contain("No batch-summary.json")); Assert.That(await File.ReadAllTextAsync(m_tablePath).ConfigureAwait(false), Is.EqualTo(before)); } private async Task WriteSummaryAsync(string batch, params (string Project, string Outcome, int TestSeconds)[] records) { string directory = Path.Combine(m_resultsPath, $"dotnet-results-{batch}", "TestResults"); Directory.CreateDirectory(directory); var document = new { projects = records.Select(record => new { project = record.Project, outcome = record.Outcome, buildSeconds = 60, testSeconds = record.TestSeconds }).ToArray() }; await File.WriteAllTextAsync( Path.Combine(directory, "batch-summary.json"), JsonSerializer.Serialize(document)).ConfigureAwait(false); } private async Task> ReadWeightsAsync() { using JsonDocument document = JsonDocument.Parse(await File.ReadAllTextAsync(m_tablePath).ConfigureAwait(false)); return document.RootElement.GetProperty("projects").EnumerateObject() .ToDictionary(project => project.Name, project => project.Value.GetInt32(), StringComparer.Ordinal); } private async Task RunScriptAsync() { string root = FindRepositoryRoot(); using var process = new Process(); process.StartInfo.FileName = "pwsh"; process.StartInfo.WorkingDirectory = root; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; PowerShellScriptOutput.ConfigureDeterministicOutput(process.StartInfo); process.StartInfo.ArgumentList.Add("-NoProfile"); process.StartInfo.ArgumentList.Add("-File"); process.StartInfo.ArgumentList.Add(Path.Combine(root, ".github", "scripts", "update-test-durations.ps1")); process.StartInfo.ArgumentList.Add("-ResultsPath"); process.StartInfo.ArgumentList.Add(m_resultsPath); process.StartInfo.ArgumentList.Add("-Source"); process.StartInfo.ArgumentList.Add("test run"); process.StartInfo.ArgumentList.Add("-DurationsPath"); process.StartInfo.ArgumentList.Add(m_tablePath); Assert.That(process.Start(), Is.True); Task standardOutput = process.StandardOutput.ReadToEndAsync(); Task standardError = process.StandardError.ReadToEndAsync(); using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(2)); await process.WaitForExitAsync(timeout.Token).ConfigureAwait(false); string output = await standardOutput.ConfigureAwait(false); string error = await standardError.ConfigureAwait(false); return new ScriptResult(process.ExitCode, PowerShellScriptOutput.Normalize(output + error)); } private static string FindRepositoryRoot() { string? current = TestContext.CurrentContext.TestDirectory; while (!string.IsNullOrWhiteSpace(current)) { if (File.Exists(Path.Combine(current, ".github", "scripts", "update-test-durations.ps1"))) { return current; } current = Directory.GetParent(current)?.FullName; } throw new InvalidOperationException("Could not find the repository root."); } private sealed record ScriptResult(int ExitCode, string Output); } } #endif