UA-.NETStandard/fuzzing/Common/Fuzz.Tests/FuzzTargetTestsBase.cs

/* ========================================================================
 * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved.
 *
 * OPC Foundation MIT License 1.00
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * The complete license agreement can be found here:
 * http://opcfoundation.org/License/MIT/1.00/
 * ======================================================================*/

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Threading.Tasks;
using NUnit.Framework;
using Opc.Ua.Tests;

namespace Opc.Ua.Fuzzing
{
    [TestFixture]
    [Category("Fuzzing")]
    public abstract class FuzzTargetTestsBase
    {
        private const int kMaxEmittedReproducers = 60;
        private const int kMaxEmittedReproducerBytes = 4096;

        public static readonly TestcaseAsset[] GoodTestcases =
        [
            .. AssetCollection<TestcaseAsset>.CreateFromFiles(
                TestUtils.EnumerateTestAssets("Testcases", "*.*"))
        ];

        public static readonly TestcaseAsset[] CrashAssets =
        [
            .. AssetCollection<TestcaseAsset>.CreateFromFiles(
                TestUtils.EnumerateTestAssets("Assets", "crash*.*"))
        ];

        public static readonly TestcaseAsset[] TimeoutAssets =
        [
            .. AssetCollection<TestcaseAsset>.CreateFromFiles(
                TestUtils.EnumerateTestAssets("Assets", "timeout*.*"))
        ];

        public static readonly TestcaseAsset[] SlowAssets =
        [
            .. AssetCollection<TestcaseAsset>.CreateFromFiles(
                TestUtils.EnumerateTestAssets("Assets", "slow*.*"))
        ];

        /// <summary>
        /// Deeply nested inputs. They are replayed through every target in a child process,
        /// because a stack overflow cannot be caught and would end the test host.
        /// </summary>
        public static readonly TestcaseAsset[] StackTestcases =
        [
            .. AssetCollection<TestcaseAsset>.CreateFromFiles(
                TestUtils.EnumerateTestAssets("StackTestcases", "*.*"))
        ];

        [DatapointSource]
        public static readonly string[] TestcaseEncoderSuffixes =
            TestUtils.DiscoverTestcaseEncoderSuffixes("Testcases");

        protected abstract Type FuzzableCodeType { get; }

        public delegate void LibFuzzTemplate(ReadOnlySpan<byte> span);

        [Theory]
        public void FuzzGoodTestcases(
            FuzzTargetFunction fuzzableCode,
            [ValueSource(nameof(GoodTestcases))] TestcaseAsset messageEncoder)
        {
            FuzzTarget(fuzzableCode, messageEncoder.Testcase);
        }

        [Theory]
        public void FuzzEmptyByteArray(FuzzTargetFunction fuzzableCode)
        {
            FuzzTarget(fuzzableCode, []);
        }

        [Theory]
        public void FuzzCrashAssets(FuzzTargetFunction fuzzableCode)
        {
            var failures = new List<string>();
            var fidelityFindings = new List<string>();
            int reproducers = 0;
            foreach (TestcaseAsset messageEncoder in CrashAssets)
            {
                try
                {
                    TestContext.Out.WriteLine(messageEncoder);
                    FuzzTarget(fuzzableCode, messageEncoder.Testcase);
                    continue;
                }
                catch (Exception ex)
                {
                    Exception cause = ex is TargetInvocationException { InnerException: not null } ?
                        ex.InnerException :
                        ex;
                    string entry = $"asset={messageEncoder} -> {cause.GetType().Name}: {cause.Message}";
                    if (IsFidelityFinding(ex) && !IsCuratedAsset(messageEncoder))
                    {
                        // Tolerated, so only the one line summary below is reported. The
                        // external corpus is private, so its bytes are never emitted for it.
                        fidelityFindings.Add(entry);
                        continue;
                    }

                    failures.Add(entry);
                    TestContext.Error.WriteLine($"Failed: {messageEncoder}\n{ex}");

                    // Crash corpora are frequently supplied by the pipeline rather than the
                    // tree, so a failure there is otherwise impossible to reproduce locally.
                    // Emit the reproducer inline, bounded so a systemic failure cannot bury
                    // the log. Only failures consume the budget, so tolerated findings cannot
                    // exhaust it ahead of a failure. A passing run emits nothing.
                    // A resource finding (allocation, time, limit) on an input from outside
                    // the tree is a resource-abuse regression the fix may not have shipped for yet, so
                    // its bytes stay out of the public log: only its name and size are logged.
                    if (IsResourceFinding(ex) && !IsCuratedAsset(messageEncoder))
                    {
                        TestContext.Error.WriteLine(
                            $"REPRODUCER {messageEncoder} withheld: resource finding on a " +
                            $"private corpus input ({messageEncoder.Testcase.Length} bytes).");
                    }
                    else if (reproducers++ < kMaxEmittedReproducers &&
                        messageEncoder.Testcase.Length <= kMaxEmittedReproducerBytes)
                    {
                        TestContext.Error.WriteLine(
                            $"REPRODUCER {messageEncoder} " +
                            Convert.ToBase64String(messageEncoder.Testcase));
                    }
                }
            }

            if (fidelityFindings.Count > 0)
            {
                // Differential targets additionally assert that a decoded value re-encodes to
                // an equivalent representation. That is a property of well formed values, and
                // an externally supplied crash corpus is a set of arbitrary mutated blobs
                // collected for other targets, so it cannot be expected to satisfy it. Report
                // the findings and keep gating those inputs on robustness only. Curated assets
                // in the tree stay strict, and continuous fuzzing still treats a fidelity
                // mismatch as a crash, so new regressions are still caught.
                TestContext.Error.WriteLine(
                    $"{fidelityFindings.Count} external crash assets reported encoding " +
                    $"fidelity findings under target '{fuzzableCode.MethodInfo.Name}'." +
                    Environment.NewLine +
                    string.Join(Environment.NewLine, fidelityFindings));
            }

            // A crash asset under Assets/crash*.* is by definition an input that
            // already produced an unhandled exception in a prior libfuzzer run.
            // The contract is: once the regression has been fixed in the fuzz
            // target (or the matching producer/decoder), replaying the asset
            // through the target must NOT throw any more. The asset's continued
            // existence in the tree therefore acts as a permanent regression
            // gate. See https://github.com/OPCFoundation/UA-.NETStandard/issues/3546
            // for the historical context that flipped this from
            // log-and-swallow to assert-and-fail.
            Assert.That(
                failures,
                Is.Empty,
                "One or more crash assets reproduced under target " +
                $"'{fuzzableCode.MethodInfo.Name}'. Each surfaced bug must be " +
                "fixed (in the decoder/encoder/parser/whitelist) before the " +
                "corresponding asset is left in place. Failures:" +
                Environment.NewLine +
                string.Join(Environment.NewLine, failures));
        }

        /// <summary>
        /// A fidelity finding means the stack stayed healthy but re-encoded a decoded value
        /// differently. It is reported rather than enforced for externally supplied inputs.
        /// <para>
        /// Matched by name because the shared exception source is linked into every fuzz
        /// target and test assembly, so the runtime types are not reference equal.
        /// </para>
        /// </summary>
        private static bool IsFidelityFinding(Exception exception)
        {
            for (Exception current = exception; current != null; current = current.InnerException!)
            {
                if (string.Equals(
                    current.GetType().FullName,
                    "Opc.Ua.Fuzzing.EncodingFidelityException",
                    StringComparison.Ordinal))
                {
                    return true;
                }
            }

            return false;
        }

        /// <summary>
        /// A resource finding means an oracle of <see cref="FuzzOracles"/> flagged the input
        /// for allocation, time or an exceeded encoding limit. It is never tolerated.
        /// </summary>
        private static bool IsResourceFinding(Exception exception)
        {
            for (Exception current = exception; current != null; current = current.InnerException!)
            {
                if (current is ResourceBudgetException)
                {
                    return true;
                }
            }

            return false;
        }

        /// <summary>
        /// Curated assets live under Assets/Repo in the tree and are always enforced strictly.
        /// Everything else in the Assets folder is overlaid by the pipeline before the build.
        /// </summary>
        private static bool IsCuratedAsset(TestcaseAsset asset)
        {
            string path = asset.Path;
            return path != null &&
                path.Replace('\\', '/')
                    .Contains("/Assets/Repo/", StringComparison.OrdinalIgnoreCase);
        }

        [Theory]
        public async Task FuzzTimeoutAssetsAsync(
            FuzzTargetFunction fuzzableCode,
            [ValueSource(nameof(TimeoutAssets))] TestcaseAsset messageEncoder)
        {
            await ReplayWithWatchdogAsync(fuzzableCode, messageEncoder.Testcase).ConfigureAwait(false);
        }

        [Theory]
        public async Task FuzzSlowAssetsAsync(
            FuzzTargetFunction fuzzableCode,
            [ValueSource(nameof(SlowAssets))] TestcaseAsset messageEncoder)
        {
            await ReplayWithWatchdogAsync(fuzzableCode, messageEncoder.Testcase).ConfigureAwait(false);
        }

        [Test]
        public async Task FuzzStackTestcasesAsync()
        {
            // The child runs each target on the small stack worker of FuzzOracles, so an
            // unguarded recursion overflows there and fails with a non-zero exit instead of
            // ending this test host.
            var failures = new List<string>();
            foreach (TestcaseAsset stackTestcase in StackTestcases)
            {
                string file = Path.Combine(Path.GetTempPath(), $"opcua-fuzz-{Guid.NewGuid():N}.bin");
                File.WriteAllBytes(file, stackTestcase.Testcase);
                try
                {
#if NETFRAMEWORK
                    string arguments = string.Empty;
#else
                    string arguments = $"\"{FuzzableCodeType.Assembly.Location}\" ";
#endif
                    ProcessStartInfo startInfo = CreateReplayStartInfo(
                        $"{arguments}{FuzzReplayProgram.ReplayAllOption} \"{file}\"");
                    (int exitCode, bool timedOut, string standardOutput, string standardError) =
                        await FuzzProcessWatchdog.RunAsync(startInfo, TimeSpan.FromSeconds(120)).ConfigureAwait(false);
                    if (timedOut || exitCode != 0)
                    {
                        failures.Add(
                            $"{stackTestcase}: exit 0x{exitCode:X8}, timed out {timedOut}" +
                            Environment.NewLine + standardOutput + Environment.NewLine + standardError);
                    }
                }
                finally
                {
                    File.Delete(file);
                }
            }

            Assert.That(failures, Is.Empty, string.Join(Environment.NewLine, failures));
        }

        [Test]
        public void RequiredCorpusAndTargetsArePresent()
        {
            Assert.That(GoodTestcases, Is.Not.Empty, "Required fuzz corpus was not copied to the test output.");
            Assert.That(CreateFuzzTargetFunctions(FuzzableCodeType), Is.Not.Empty);
        }

        protected static FuzzTargetFunction[] CreateFuzzTargetFunctions(Type fuzzableCodeType)
        {
            return
            [
                .. fuzzableCodeType
                    .GetMethods(BindingFlags.Static | BindingFlags.Public)
                    .Where(f => f.ReturnType == typeof(void) &&
                        !f.ContainsGenericParameters &&
                        f.GetParameters().Length == 1 &&
                        (f.GetParameters()[0].ParameterType == typeof(Stream) ||
                         f.GetParameters()[0].ParameterType == typeof(string) ||
                         f.GetParameters()[0].ParameterType == typeof(ReadOnlySpan<byte>)))
                    .Select(f => new FuzzTargetFunction(f))
            ];
        }

        protected virtual void OnFuzzTargetSetup(ITelemetryContext telemetry)
        {
        }

        private async Task ReplayWithWatchdogAsync(FuzzTargetFunction target, byte[] data)
        {
            string file = Path.Combine(Path.GetTempPath(), $"opcua-fuzz-{Guid.NewGuid():N}.bin");
            File.WriteAllBytes(file, data);
            try
            {
                string assembly = FuzzableCodeType.Assembly.Location;
#if NETFRAMEWORK
                string executable = assembly;
                string arguments = string.Empty;
#else
                string executable = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet";
                string arguments = $"\"{assembly}\" ";
#endif
                ProcessStartInfo startInfo = CreateReplayStartInfo(
                    $"{arguments}--fuzz-replay {target.MethodInfo.Name} \"{file}\"");
                (int exitCode, bool timedOut, string standardOutput, string standardError) =
                    await FuzzProcessWatchdog.RunAsync(startInfo, TimeSpan.FromSeconds(30)).ConfigureAwait(false);
                Assert.That(timedOut, Is.False, $"Replay exceeded the process budget: {target.MethodInfo.Name}");
                Assert.That(exitCode, Is.Zero, standardOutput + Environment.NewLine + standardError);
            }
            finally
            {
                File.Delete(file);
            }
        }

        private static ProcessStartInfo CreateReplayStartInfo(string arguments)
        {
#if NETFRAMEWORK
            string executable = typeof(FuzzTargetTestsBase).Assembly.Location;
#else
            string executable = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet";
#endif
            return new ProcessStartInfo
            {
                FileName = executable,
                Arguments = arguments,
                UseShellExecute = false,
                RedirectStandardOutput = true,
                RedirectStandardError = true,
                CreateNoWindow = true,
                WorkingDirectory = AppContext.BaseDirectory
            };
        }

        [TestCase("MissingFuzzTarget \"no-such-input\"")]
        [TestCase("OnlyOneArgument")]
        public async Task MalformedReplayRequestFailsInsteadOfReportingSuccessAsync(string replayArguments)
        {
            // The timeout/slow regressions assert a zero exit, so a replay child that
            // cannot run the requested input must fail rather than fall through to the
            // benchmark host and report a pass for an input that was never replayed.
#if NETFRAMEWORK
            string prefix = string.Empty;
#else
            string prefix = $"\"{typeof(FuzzTargetTestsBase).Assembly.Location}\" ";
#endif
            ProcessStartInfo startInfo = CreateReplayStartInfo(
                $"{prefix}--fuzz-replay {replayArguments}");

            (int exitCode, bool timedOut, _, _) =
                await FuzzProcessWatchdog.RunAsync(startInfo, TimeSpan.FromSeconds(30)).ConfigureAwait(false);

            Assert.That(timedOut, Is.False);
            Assert.That(exitCode, Is.Not.Zero);
        }

        private void FuzzTarget(FuzzTargetFunction fuzzableCode, byte[] blob)
        {
            ITelemetryContext telemetry = NUnitTelemetryContext.Create();
            OnFuzzTargetSetup(telemetry);

            ParameterInfo[] parameters = fuzzableCode.MethodInfo.GetParameters();
            if (parameters.Length != 1)
            {
                throw new InvalidOperationException(
                    "Fuzzable function must have exactly one parameter.");
            }
            // The same stack and time oracles as the fuzzers, so a curated reproducer of a
            // resource finding fails here as it did under libFuzzer.
            FuzzOracles.RunTarget(
                fuzzableCode.MethodInfo.Name,
                blob.Length,
                () => InvokeFuzzTarget(fuzzableCode, parameters[0].ParameterType, blob));
        }

        private static void InvokeFuzzTarget(FuzzTargetFunction fuzzableCode, Type parameterType, byte[] blob)
        {
            try
            {
                if (parameterType == typeof(string))
                {
                    string text = Encoding.UTF8.GetString(blob);
                    _ = fuzzableCode.MethodInfo.Invoke(null, [text]);
                }
                else if (typeof(Stream).IsAssignableFrom(parameterType))
                {
                    using var stream = new MemoryStream(blob);
                    _ = fuzzableCode.MethodInfo.Invoke(null, [stream]);
                }
                else if (parameterType == typeof(ReadOnlySpan<byte>))
                {
                    var span = new ReadOnlySpan<byte>(blob);
#if NET8_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER
                    LibFuzzTemplate fuzzFunction = fuzzableCode.MethodInfo
                        .CreateDelegate<LibFuzzTemplate>();
#else
                    var fuzzFunction = (LibFuzzTemplate)fuzzableCode.MethodInfo
                        .CreateDelegate(typeof(LibFuzzTemplate));
#endif
                    fuzzFunction(span);
                }
                else
                {
                    throw new InvalidOperationException("Unsupported fuzz target signature.");
                }
            }
            catch (TargetInvocationException ex) when (ex.InnerException != null)
            {
                // Report the finding of the target, not the reflection wrapper, so the
                // resource and fidelity classification sees the exception the target threw.
                ExceptionDispatchInfo.Capture(ex.InnerException).Throw();
                throw;
            }
        }
    }
}