/* ======================================================================== * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Opc.Ua; using Opc.Ua.AI.Server; using Opc.Ua.Vision; using Opc.Ua.Vision.Server; namespace Vision.VisualInspectionCell { internal sealed class VisualInspectionFeedbackSink : IVisionFeedbackSink { public VisualInspectionFeedbackSink( VisualInspectionResultPublisher publisher, AINodeManagerRegistry aiRegistry, InspectionVerdictPolicy verdictPolicy, TimeProvider timeProvider, ILogger logger) { m_publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); m_aiRegistry = aiRegistry ?? throw new ArgumentNullException(nameof(aiRegistry)); m_verdictPolicy = verdictPolicy ?? throw new ArgumentNullException(nameof(verdictPolicy)); m_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); m_logger = logger ?? throw new ArgumentNullException(nameof(logger)); } public void Attach(VisualInspectionTarget target) { if (target == null) { throw new ArgumentNullException(nameof(target)); } if (Interlocked.CompareExchange(ref m_target, target, null) != null) { throw new InvalidOperationException("The feedback sink is already attached."); } if (m_logger.IsEnabled(LogLevel.Information)) { m_logger.FeedbackAttached(target.PipelineNodeId.ToString()); } } public async ValueTask SubmitDetectionsAsync( VisionSubmitDetectionsRequest request, CancellationToken cancellationToken) { if (request == null) { throw new ArgumentNullException(nameof(request)); } cancellationToken.ThrowIfCancellationRequested(); if (request.SceneIsEmpty && request.Purpose == VisionFeedbackPurposeEnum.GroundTruthLabel) { await RecordLearningSampleAsync( StableSampleId(request.Pipeline, FrameKey(request.FrameReference, "empty-scene"), "scene-empty"), AILearningSampleKind.Negative, cancellationToken).ConfigureAwait(false); } else if (request.Purpose == VisionFeedbackPurposeEnum.GroundTruthLabel) { await RecordLearningSampleAsync( StableSampleId(request.Pipeline, FrameKey(request.FrameReference, "geometry"), "geometry"), AILearningSampleKind.Positive, cancellationToken).ConfigureAwait(false); } return ServiceResult.Good; } public async ValueTask SubmitInspectionResultAsync( VisionSubmitInspectionResultRequest request, CancellationToken cancellationToken) { if (request == null) { throw new ArgumentNullException(nameof(request)); } cancellationToken.ThrowIfCancellationRequested(); VisualInspectionTarget target = RequireTarget(); if (!request.Pipeline.Equals(target.PipelineNodeId)) { return new ServiceResult(StatusCodes.BadNodeIdUnknown, LocalizedText.From("The pipeline node id does not match the attached inspection pipeline.")); } string resultId = string.IsNullOrEmpty(request.ResultId) ? StableResultId(request.Pipeline, request.Characteristics) : request.ResultId; if (!TryGetImageReference(resultId, out VisionImageReferenceDataType? frameReference)) { return new ServiceResult(StatusCodes.BadInvalidState, LocalizedText.From("SubmitImageReference must provide provenance before publishing a result.")); } VisionResultEvaluationEnum evaluation = m_verdictPolicy.JudgeCharacteristics(request.Characteristics); PublishedInspectionResult published = await m_publisher.PublishAsync( target, resultId, TimestampOrNow(frameReference.Timestamp), evaluation, request.Characteristics, "agent-edge-off-server", frameReference, string.Empty, cancellationToken).ConfigureAwait(false); if (evaluation == VisionResultEvaluationEnum.NotDecidable) { // An off-server agent supplied evidence, but production policy still remains external: // the server only exposes the dialog and waits for the operator's OPC UA response. m_operatorDialog?.RequestDisposition(published); } m_logger.AgentInspectionPublished(resultId, evaluation); return ServiceResult.Good; } public async ValueTask SubmitCorrectionAsync( VisionSubmitCorrectionRequest request, CancellationToken cancellationToken) { if (request == null) { throw new ArgumentNullException(nameof(request)); } cancellationToken.ThrowIfCancellationRequested(); VisualInspectionTarget target = RequireTarget(); if (!request.Pipeline.Equals(target.PipelineNodeId)) { return new ServiceResult(StatusCodes.BadNodeIdUnknown, LocalizedText.From("The pipeline node id does not match the attached inspection pipeline.")); } AILearningSampleKind kind = request.RetractAll ? AILearningSampleKind.Negative : AILearningSampleKind.Positive; string sampleId = GroundTruthSampleId(request.Pipeline, request.ResultId); await RecordLearningSampleAsync(sampleId, kind, cancellationToken).ConfigureAwait(false); if (request.CorrectedCharacteristics.Count > 0) { string resultId = FormattableString.Invariant($"correction-{Sanitize(request.ResultId)}"); VisionImageReferenceDataType frameReference = TryGetImageReference(request.ResultId, out var image) ? image : FrameFromPublished(request.ResultId); await m_publisher.PublishAsync( target, resultId, TimestampOrNow(frameReference.Timestamp), m_verdictPolicy.JudgeCharacteristics(request.CorrectedCharacteristics), request.CorrectedCharacteristics, "operator-ground-truth", frameReference, FixtureFromPublished(request.ResultId), cancellationToken).ConfigureAwait(false); } m_logger.CorrectionRecorded(sampleId, kind); return ServiceResult.Good; } public ValueTask SubmitImageReferenceAsync( VisionSubmitImageReferenceRequest request, CancellationToken cancellationToken) { if (request == null) { throw new ArgumentNullException(nameof(request)); } cancellationToken.ThrowIfCancellationRequested(); if (string.IsNullOrEmpty(request.ResultId)) { return ValueTask.FromResult(new ServiceResult( StatusCodes.BadInvalidArgument, LocalizedText.From("An image reference must name the result it belongs to."))); } lock (m_lock) { m_imageReferences[request.ResultId] = request.Image; } return ValueTask.FromResult(ServiceResult.Good); } public void AttachOperatorDialog(OperatorDialogController operatorDialog) { m_operatorDialog = operatorDialog ?? throw new ArgumentNullException(nameof(operatorDialog)); } public ValueTask HandleOperatorDispositionAsync( PublishedInspectionResult result, OperatorDisposition disposition, CancellationToken cancellationToken) { if (result == null) { throw new ArgumentNullException(nameof(result)); } return HandleOperatorDispositionCoreAsync(result, disposition, cancellationToken); } private async ValueTask RecordLearningSampleAsync( string sampleId, AILearningSampleKind kind, CancellationToken cancellationToken) { AINodeManager? manager = m_aiRegistry.NodeManager; if (manager == null) { m_logger.LearningSampleSkipped(sampleId); return; } bool added = await manager.RecordLearningSampleAsync(sampleId, kind, cancellationToken) .ConfigureAwait(false); m_logger.LearningSampleRecorded(sampleId, kind, added); } private VisualInspectionTarget RequireTarget() { return m_target ?? throw new InvalidOperationException("The feedback sink is not attached."); } private async ValueTask HandleOperatorDispositionCoreAsync( PublishedInspectionResult result, OperatorDisposition disposition, CancellationToken cancellationToken) { VisualInspectionTarget target = RequireTarget(); VisionResultEvaluationEnum evaluation = disposition switch { OperatorDisposition.AcceptAsOk => VisionResultEvaluationEnum.Ok, OperatorDisposition.AcceptAsNotOk => VisionResultEvaluationEnum.NotOk, _ => VisionResultEvaluationEnum.NotDecidable }; if (evaluation == VisionResultEvaluationEnum.NotDecidable) { m_logger.OperatorDispositionIgnored(result.ResultId, disposition); return; } AILearningSampleKind kind = disposition == OperatorDisposition.AcceptAsNotOk ? AILearningSampleKind.Negative : AILearningSampleKind.Positive; string sampleId = GroundTruthSampleId(target.PipelineNodeId, result.ResultId); await RecordLearningSampleAsync(sampleId, kind, cancellationToken).ConfigureAwait(false); string correctionId = FormattableString.Invariant( $"operator-{Sanitize(result.ResultId)}-{disposition}"); await m_publisher.PublishAsync( target, correctionId, TimestampOrNow(result.FrameReference.Timestamp), evaluation, result.Characteristics, "operator-ground-truth", result.FrameReference, result.FixtureName, cancellationToken).ConfigureAwait(false); m_logger.OperatorDispositionRecorded(result.ResultId, disposition, sampleId, kind); } private static string StableSampleId(NodeId pipeline, string resultId, string purpose) { return FormattableString.Invariant($"{pipeline}|{resultId}|{purpose}"); } /// /// Identifies the ground truth held for one inspection result. /// /// /// A correction can arrive by two routes - an operator answering the /// disposition dialog, or a caller invoking SubmitCorrection - and /// both describe the same decision about the same frame. Keying the sample /// on the result alone makes the second arrival a duplicate that the /// accounting rejects. Keying it on the route, as this once did, made one /// human decision count twice and inflated the very number section 9.4 /// asks a Server to keep honestly. /// private static string GroundTruthSampleId(NodeId pipeline, string resultId) { return StableSampleId(pipeline, resultId, "ground-truth"); } private static string StableResultId( NodeId pipeline, ArrayOf characteristics) { string key = characteristics.Count == 0 ? "empty" : characteristics[0].CharacteristicId + ":" + characteristics[0].Actual.ToString(System.Globalization.CultureInfo.InvariantCulture); return FormattableString.Invariant($"agent-insp-{Sanitize(pipeline.ToString())}-{Sanitize(key)}"); } private static string FrameKey(VisionImageReferenceDataType frame, string fallback) { return string.IsNullOrEmpty(frame?.Uri) ? fallback : frame.Uri; } private bool TryGetImageReference( string resultId, [NotNullWhen(true)] out VisionImageReferenceDataType? image) { lock (m_lock) { return m_imageReferences.TryGetValue(resultId, out image); } } private VisionImageReferenceDataType FrameFromPublished(string resultId) { return m_publisher.TryGetPublished(resultId, out PublishedInspectionResult? published) && published != null ? published.FrameReference : new VisionImageReferenceDataType { Timestamp = TimestampOrNow(DateTimeUtc.MinValue) }; } private string FixtureFromPublished(string resultId) { return m_publisher.TryGetPublished(resultId, out PublishedInspectionResult? published) && published != null ? published.FixtureName : string.Empty; } private DateTimeUtc TimestampOrNow(DateTimeUtc timestamp) { return timestamp.IsNull ? DateTimeUtc.From(m_timeProvider.GetUtcNow()) : timestamp; } private static string Sanitize(string value) { return string.IsNullOrEmpty(value) ? "unknown" : value.Replace('-', '_'); } private readonly VisualInspectionResultPublisher m_publisher; private readonly AINodeManagerRegistry m_aiRegistry; private readonly InspectionVerdictPolicy m_verdictPolicy; private readonly TimeProvider m_timeProvider; private readonly ILogger m_logger; private readonly Lock m_lock = new(); private readonly Dictionary m_imageReferences = []; private VisualInspectionTarget? m_target; private OperatorDialogController? m_operatorDialog; } internal static partial class VisualInspectionFeedbackSinkLog { [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 1, Level = LogLevel.Information, Message = "Visual inspection feedback sink attached to pipeline {PipelineNodeId}.")] public static partial void FeedbackAttached(this ILogger logger, string pipelineNodeId); [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 2, Level = LogLevel.Information, Message = "Published off-server inspection result {ResultId}: {Verdict}.")] public static partial void AgentInspectionPublished( this ILogger logger, string resultId, VisionResultEvaluationEnum verdict); [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 3, Level = LogLevel.Information, Message = "Recorded correction sample {SampleId} ({Kind}).")] public static partial void CorrectionRecorded( this ILogger logger, string sampleId, AILearningSampleKind kind); [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 4, Level = LogLevel.Information, Message = "Learning sample {SampleId} ({Kind}) added={Added}.")] public static partial void LearningSampleRecorded( this ILogger logger, string sampleId, AILearningSampleKind kind, bool added); [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 5, Level = LogLevel.Warning, Message = "AI node manager not available; learning sample {SampleId} was not counted.")] public static partial void LearningSampleSkipped( this ILogger logger, string sampleId); [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 6, Level = LogLevel.Information, Message = "Operator disposition {Disposition} for {ResultId} recorded as {SampleId} ({Kind}).")] public static partial void OperatorDispositionRecorded( this ILogger logger, string resultId, OperatorDisposition disposition, string sampleId, AILearningSampleKind kind); [LoggerMessage(EventId = VisualInspectionCellEventIds.Feedback + 7, Level = LogLevel.Information, Message = "Operator disposition {Disposition} for result {ResultId} is operational only; no ground-truth sample was recorded.")] public static partial void OperatorDispositionIgnored( this ILogger logger, string resultId, OperatorDisposition disposition); } }