/* ======================================================================== * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Collections.Generic; using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.AI; using Moq; using NUnit.Framework; using Opc.Ua.AI.Inference; namespace Opc.Ua.AI.Tests { /// /// Covers the projection. /// /// /// The point of routing through Microsoft.Extensions.AI is that one /// abstraction covers a hosted service and an on-device runtime, which is the /// property clause 8.1 asserts about where inference runs. These tests use a /// stub , so they exercise the projection both ways /// without a model, a network or a GPU - which is also what lets them run in /// CI. What they mostly check is that nothing is invented: usage the client /// did not report must not be estimated, and a model it did not name must not /// be guessed at. /// [TestFixture] public sealed class ChatClientInferenceBackendTests { [Test] public async Task InvokeProjectsMessagesAndReportsWhatTheClientAnswered() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "pong")) { ModelId = "served-model", FinishReason = ChatFinishReason.Stop, Usage = new UsageDetails { InputTokenCount = 11, OutputTokenCount = 22, TotalTokenCount = 33 } }); using var backend = new ChatClientInferenceBackend(client, InferenceSite.EdgeOffServer); InferenceResult result = await backend.InvokeAsync( Request("""{"messages":[{"role":"user","content":"ping"}]}""", "asked-model"), CancellationToken.None).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(result.Ok, Is.True); Assert.That(Encoding.UTF8.GetString(result.Payload.Span), Is.EqualTo("pong")); Assert.That(result.ModelUsed, Is.EqualTo("served-model"), "The model that actually answered is not always the one asked for, " + "and a caller that cannot see the substitution cannot tell a degraded " + "answer from a good one."); Assert.That(result.InputUnits, Is.EqualTo(11UL)); Assert.That(result.OutputUnits, Is.EqualTo(22UL)); Assert.That(result.TotalUnits, Is.EqualTo(33UL)); Assert.That(result.Finish, Is.EqualTo(InferenceFinish.Stop)); Assert.That(client.LastMessages, Has.Count.EqualTo(1)); Assert.That(client.LastMessages![0].Role, Is.EqualTo(ChatRole.User)); Assert.That(client.LastMessages![0].Text, Is.EqualTo("ping")); Assert.That(client.LastOptions?.ModelId, Is.EqualTo("asked-model")); }); } [Test] public async Task InvokeReportsTheRequestedModelWhenTheClientNamesNone() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client); InferenceResult result = await backend.InvokeAsync( Request("""{"messages":[{"role":"user","content":"hi"}]}""", "asked-model"), CancellationToken.None).ConfigureAwait(false); // The request named a model and nothing observed contradicts it, so it // is reported rather than second-guessed. Assert.That(result.ModelUsed, Is.EqualTo("asked-model")); } [Test] public async Task InvokeKeepsPlainStringContentAsATextMessage() { List? captured = null; Mock client = CreateCapturingClient((messages, _, _) => captured = new List(messages)); using var backend = new ChatClientInferenceBackend(client.Object); await backend.InvokeAsync( Request("""{"messages":[{"role":"user","content":"ping"}]}"""), CancellationToken.None).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(captured, Is.Not.Null); Assert.That(captured![0].Role, Is.EqualTo(ChatRole.User)); Assert.That(captured[0].Text, Is.EqualTo("ping")); }); } [Test] public async Task InvokeProjectsTextAndDataImagePartsToMultimodalContent() { byte[] imageBytes = [1, 2, 3, 4]; string image = Convert.ToBase64String(imageBytes); List? captured = null; Mock client = CreateCapturingClient((messages, _, _) => captured = new List(messages)); using var backend = new ChatClientInferenceBackend(client.Object); await backend.InvokeAsync( Request( """ {"messages":[{"role":"user","content":[ {"type":"text","text":"Measure the bore diameter."}, {"type":"image_url","image_url":{"url":"data:image/png;base64,$IMAGE$","detail":"high"}} ]}]} """.Replace("$IMAGE$", image, StringComparison.Ordinal)), CancellationToken.None).ConfigureAwait(false); IList contents = captured![0].Contents; var text = (TextContent)contents[0]; var data = (DataContent)contents[1]; Assert.Multiple(() => { Assert.That(contents, Has.Count.EqualTo(2)); Assert.That(text.Text, Is.EqualTo("Measure the bore diameter.")); Assert.That(data.MediaType, Is.EqualTo("image/png")); Assert.That(data.Data.ToArray(), Is.EqualTo(imageBytes).AsCollection); Assert.That(data.AdditionalProperties?["detail"], Is.EqualTo("high")); }); } [Test] public async Task InvokeProjectsRemoteImagePartToUriContent() { List? captured = null; Mock client = CreateCapturingClient((messages, _, _) => captured = new List(messages)); using var backend = new ChatClientInferenceBackend(client.Object); await backend.InvokeAsync( Request(""" {"messages":[{"role":"user","content":[ {"type":"image_url","image_url":{"url":"https://example.test/frame.png"}} ]}]} """), CancellationToken.None).ConfigureAwait(false); var uri = (UriContent)captured![0].Contents[0]; Assert.Multiple(() => { Assert.That(uri.Uri, Is.EqualTo(new Uri("https://example.test/frame.png"))); Assert.That(uri.MediaType, Is.EqualTo("image/png")); }); } [Test] public async Task InvokeRefusesMalformedImageDataUriBeforeCallingTheClient() { List? captured = null; Mock client = CreateCapturingClient((messages, _, _) => captured = new List(messages)); using var backend = new ChatClientInferenceBackend(client.Object); InferenceResult result = await backend.InvokeAsync( Request(""" {"messages":[{"role":"user","content":[ {"type":"image_url","image_url":{"url":"data:image/png;base64,not-base64"}} ]}]} """), CancellationToken.None).ConfigureAwait(false); // A malformed image is refused before inference so callers do not get // an answer about a picture the model never received. Assert.Multiple(() => { Assert.That(result.Ok, Is.False); Assert.That(result.Message, Does.Contain("base64")); Assert.That(captured, Is.Null); }); } [Test] [TestCase("data:foo;base64,AAAA", TestName = "MediaTypeWithoutASubtype")] [TestCase("data:image;base64,AAAA", TestName = "MediaTypeMissingItsSubtype")] [TestCase("data:image//png;base64,AAAA", TestName = "MediaTypeWithTwoSeparators")] public async Task InvokeRefusesAnInvalidImageMediaTypeAsAStructuredResult(string url) { List? captured = null; Mock client = CreateCapturingClient((messages, _, _) => captured = new List(messages)); using var backend = new ChatClientInferenceBackend(client.Object); InferenceResult result = await backend.InvokeAsync( Request( "{\"messages\":[{\"role\":\"user\",\"content\":[" + "{\"type\":\"image_url\",\"image_url\":{\"url\":\"" + url + "\"}}" + "]}]}"), CancellationToken.None).ConfigureAwait(false); // DataContent validates the media type itself and throws // ArgumentException, not FormatException. Left uncaught that escapes // InvokeAsync entirely, so the caller gets a bare Bad status carrying an // internal exception message rather than a refusal naming the problem - // and on the chunked-transfer path the transfer is stranded Executing, // holding a concurrency slot until it expires. Assert.Multiple(() => { Assert.That(result.Ok, Is.False); Assert.That(result.Finish, Is.EqualTo(InferenceFinish.Error)); Assert.That(result.Message, Does.Contain("media type")); Assert.That(captured, Is.Null); }); } [Test] public async Task InvokeRefusesANonStringRoleAsAStructuredResult() { List? captured = null; Mock client = CreateCapturingClient( (messages, _, _) => captured = new List(messages)); using var backend = new ChatClientInferenceBackend(client.Object); InferenceResult result = await backend.InvokeAsync( Request("""{"messages":[{"role":5,"content":"hi"}]}"""), CancellationToken.None).ConfigureAwait(false); // JsonElement.GetString throws on a non-string element, which would // escape the handler that turns a bad payload into a refusal. Assert.Multiple(() => { Assert.That(result.Ok, Is.False); Assert.That(result.Message, Does.Contain("role")); Assert.That(captured, Is.Null); }); } [Test] public async Task InvokeReportsZeroUnitsWhenTheClientReportsNoUsage() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client); InferenceResult result = await backend.InvokeAsync( Request("""{"messages":[{"role":"user","content":"hi"}]}"""), CancellationToken.None).ConfigureAwait(false); // Estimating usage a backend did not report would produce a number that // looks metered and is not, and it would be billed against. Assert.Multiple(() => { Assert.That(result.InputUnits, Is.Zero); Assert.That(result.OutputUnits, Is.Zero); Assert.That(result.TotalUnits, Is.Zero); }); } [Test] public async Task InvokeRefusesAPayloadThatIsNotValidJson() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "unused"))); using var backend = new ChatClientInferenceBackend(client); InferenceResult result = await backend.InvokeAsync( Request("{ not json"), CancellationToken.None).ConfigureAwait(false); // Malformed input from a caller is expected rather than exceptional, // and the reason is more use than a transport fault. Assert.Multiple(() => { Assert.That(result.Ok, Is.False); Assert.That(result.Finish, Is.EqualTo(InferenceFinish.Error)); Assert.That(result.Message, Does.Contain("not a valid chat completions body")); Assert.That(client.LastMessages, Is.Null, "The client must not be reached."); }); } [Test] public async Task InvokeRefusesAPayloadCarryingNoMessages() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "unused"))); using var backend = new ChatClientInferenceBackend(client); InferenceResult result = await backend.InvokeAsync( Request("""{"messages":[]}"""), CancellationToken.None).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(result.Ok, Is.False); Assert.That(result.Message, Does.Contain("no messages")); Assert.That(client.LastMessages, Is.Null); }); } [Test] public async Task InvokeMapsTheRolesTheContractDefines() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client); await backend.InvokeAsync( Request(""" {"messages":[ {"role":"system","content":"s"}, {"role":"user","content":"u"}, {"role":"assistant","content":"a"}, {"role":"tool","content":"t"}]} """), CancellationToken.None).ConfigureAwait(false); Assert.That( new[] { client.LastMessages![0].Role, client.LastMessages![1].Role, client.LastMessages![2].Role, client.LastMessages![3].Role }, Is.EqualTo(new[] { ChatRole.System, ChatRole.User, ChatRole.Assistant, ChatRole.Tool }).AsCollection); } [Test] public async Task InvokeAppliesTheCallParametersTheBackendSupports() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client); await backend.InvokeAsync( Request( """{"messages":[{"role":"user","content":"hi"}]}""", parameters: new Dictionary { ["temperature"] = "0.25", ["max_tokens"] = "64", ["top_p"] = "0.9" }), CancellationToken.None).ConfigureAwait(false); // Parsed invariantly: on a machine whose locale writes 0,25 a // culture-sensitive parse silently drops the decimal point. Assert.Multiple(() => { Assert.That(client.LastOptions!.Temperature, Is.EqualTo(0.25f)); Assert.That(client.LastOptions!.MaxOutputTokens, Is.EqualTo(64)); Assert.That(client.LastOptions!.TopP, Is.EqualTo(0.9f)); }); } [Test] public void InvokeRefusesACallParameterItCannotHonour() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client); // A caller whose parameter was silently dropped believes it took // effect, and there is no later point at which it can find out. Assert.That( async () => await backend.InvokeAsync( Request( """{"messages":[{"role":"user","content":"hi"}]}""", parameters: new Dictionary { ["seed"] = "1" }), CancellationToken.None).ConfigureAwait(false), Throws.ArgumentException); } [Test] public async Task ProbeReportsUnreachableRatherThanRaisingWhenTheClientFails() { var client = new StubChatClient(new InvalidOperationException("no route to host")); using var backend = new ChatClientInferenceBackend(client); BackendProbe probe = await backend.ProbeAsync(CancellationToken.None) .ConfigureAwait(false); // A probe exists so a commissioning engineer learns the endpoint is // wrong before a deployment depends on it, so it reports rather than // throws. Assert.Multiple(() => { Assert.That(probe.Reachable, Is.False); Assert.That(probe.Detail, Does.Contain("no route to host")); }); } [Test] public async Task ProbeReportsReachableAndNamesTheModelThatAnswered() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "pong")) { ModelId = "served-model" }); using var backend = new ChatClientInferenceBackend(client); BackendProbe probe = await backend.ProbeAsync(CancellationToken.None) .ConfigureAwait(false); Assert.Multiple(() => { Assert.That(probe.Reachable, Is.True); Assert.That(probe.Detail, Is.EqualTo("served-model")); }); } [Test] public async Task ListModelsFiltersAndBoundsTheSuppliedCatalogue() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend( client, InferenceSite.OnServer, new[] { new BackendModel { Name = "alpha-small" }, new BackendModel { Name = "alpha-large" }, new BackendModel { Name = "beta" } }); IReadOnlyList all = await backend .ListModelsAsync(null, 0, CancellationToken.None).ConfigureAwait(false); IReadOnlyList filtered = await backend .ListModelsAsync("alpha", 0, CancellationToken.None).ConfigureAwait(false); IReadOnlyList bounded = await backend .ListModelsAsync(null, 2, CancellationToken.None).ConfigureAwait(false); Assert.Multiple(() => { Assert.That(all, Has.Count.EqualTo(3)); Assert.That(filtered, Has.Count.EqualTo(2)); Assert.That(bounded, Has.Count.EqualTo(2)); }); } [Test] public void SiteIsWhatTheHostStatedRatherThanSomethingInferred() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client, InferenceSite.OnServer); // An IChatClient over a local runtime and one over a hosted service are // the same type, so the client cannot be asked where it runs. Assert.That(backend.Site, Is.EqualTo(InferenceSite.OnServer)); } [Test] public void ConstructorRefusesANullClient() { Assert.That( () => new ChatClientInferenceBackend(null!), Throws.ArgumentNullException); } [Test] public void InvokeRefusesANullRequest() { var client = new StubChatClient( new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); using var backend = new ChatClientInferenceBackend(client); Assert.That( async () => await backend.InvokeAsync(null!, CancellationToken.None) .ConfigureAwait(false), Throws.ArgumentNullException); } private static InferenceRequest Request( string body, string model = "", IReadOnlyDictionary? parameters = null) { return new InferenceRequest { Model = model, Payload = Encoding.UTF8.GetBytes(body), ContentType = "application/json", Parameters = parameters ?? new Dictionary() }; } private static Mock CreateCapturingClient( Action, ChatOptions?, CancellationToken> callback) { var client = new Mock(); client .Setup(c => c.GetResponseAsync( It.IsAny>(), It.IsAny(), It.IsAny())) .Callback(callback) .ReturnsAsync(new ChatResponse(new ChatMessage(ChatRole.Assistant, "ok"))); return client; } private sealed class StubChatClient : IChatClient { private readonly ChatResponse? m_response; private readonly Exception? m_failure; public StubChatClient(ChatResponse response) { m_response = response; } public StubChatClient(Exception failure) { m_failure = failure; } public List? LastMessages { get; private set; } public ChatOptions? LastOptions { get; private set; } public Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { LastMessages = new List(messages); LastOptions = options; if (m_failure != null) { throw m_failure; } return Task.FromResult(m_response!); } public IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { throw new NotSupportedException(); } public object? GetService(Type serviceType, object? serviceKey = null) { return null; } public void Dispose() { } } } }