/* ======================================================================== * Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved. * * OPC Foundation MIT License 1.00 * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, * copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following * conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * * The complete license agreement can be found here: * http://opcfoundation.org/License/MIT/1.00/ * ======================================================================*/ using System; using System.Collections.Generic; using System.Globalization; using System.Threading; using System.Threading.Tasks; using Opc.Ua.Client; using Opc.Ua.Mcp.Tools; using Opc.Ua.Robotics.Client.Intent; using Opc.Ua.RobotIntent; using Opc.Ua.Vision; using Opc.Ua.Vision.Client; namespace Opc.Ua.Mcp { /// /// Runs one Vision inference and turns the detection it selects into exactly /// one piece of Robot Intent work on the same OPC UA session. /// /// /// The manager is the cross-companion helper the robotics_vision_pick /// tool delegates to. It composes the Vision and Robot Intent client SDKs /// directly - it never calls another MCP tool - so the same orchestration is /// available to application code that does not host an MCP server. It is /// read-only towards command authority: it never requests or releases /// control, never waits for completion, never retries, and never cancels. /// Every refusal the server reports is returned verbatim. /// public sealed class VisionGuidedRoboticsManager { /// /// Initializes the manager over the MCP session manager and the Robot /// Intent manager. This is the constructor the container resolves, and /// it is equally usable as a direct-construction fallback. /// /// /// The session manager that owns the named OPC UA sessions. /// /// /// The Robot Intent manager used to resolve the controller. /// /// /// Any argument is null. /// public VisionGuidedRoboticsManager( OpcUaSessionManager sessionManager, RoboticsIntentManager roboticsManager) { m_sessionManager = sessionManager ?? throw new ArgumentNullException(nameof(sessionManager)); Robotics = roboticsManager ?? throw new ArgumentNullException(nameof(roboticsManager)); } /// /// Gets the Robot Intent manager the controller is resolved through. /// public RoboticsIntentManager Robotics { get; } /// /// Creates a Vision client over the very same named session the Robot /// Intent clients are created on, so perception and actuation always /// observe one server through one session. /// /// /// Session name to use; defaults to the only active session. /// public VisionClient CreateVisionClient(string? sessionName = null) { ISession session = m_sessionManager.GetSessionOrThrow(sessionName); return new VisionClient(session, m_sessionManager.Telemetry); } /// /// Runs one Vision inference, selects one detection deterministically, /// and submits either a single Pick intent or a two-step Pick/Place /// mission exactly once. /// /// /// The vision-guided pick request. /// /// /// Cancels the operation. /// /// /// is null. /// /// /// A request field is empty, out of range, non-finite, or conflicts /// with another field. /// /// /// The Vision result could not be resolved, or no detection matched. /// public async Task PickAsync( VisionGuidedPickRequest request, CancellationToken ct = default) { ValidateRequest(request); RoboticsResolutionContext context = await RoboticsResolutionContext.CreateAsync( Robotics, request.Controller, request.SessionName, ct).ConfigureAwait(false); VisionPickObservation observation = await ObserveAsync(request, ct).ConfigureAwait(false); return await SubmitAsync(request, context, observation, ct).ConfigureAwait(false); } /// /// Runs the Vision half of a vision-guided pick on the same session: /// resolves the pipeline, runs one bounded one-shot inference, and reads /// the full detection snapshot when the bounded summary is incomplete. /// internal async Task ObserveAsync( VisionGuidedPickRequest request, CancellationToken ct) { VisionClient client = CreateVisionClient(request.SessionName); VisionNodeEntry entry = await client.ResolvePipelineAsync(request.Pipeline, ct) .ConfigureAwait(false); VisionPipelineClient pipeline = client.Pipeline(entry.NodeId); VisionInferenceService inference = client.Inference(); VisionInferenceResult result = await inference.RunOneShotAsync( pipeline, entry.BrowseName.Name, VisionResultDetail.Summary, VisionExpectedResultKind.Detection, kMaxSummaryItems, ct).ConfigureAwait(false); return await ResolveObservationAsync( result, token => client.Result(result.ResultNodeId).ReadDetectionAsync(token), ct).ConfigureAwait(false); } /// /// Projects one inference result into the detection set selection runs /// over, reading the full snapshot when the bounded summary truncated it. /// /// internal static async Task ResolveObservationAsync( VisionInferenceResult result, Func> readFullSnapshot, CancellationToken ct) { ArgumentNullException.ThrowIfNull(result); ArgumentNullException.ThrowIfNull(readFullSnapshot); if (!result.Resolved || result.ResultNodeId.IsNull) { // All concatenated operands must remain interpolated for string.Create handler binding. // TODO: Remove when RCS1214 preserves interpolated-string-handler overload binding. #pragma warning disable RCS1214 throw new InvalidOperationException(string.Create(CultureInfo.InvariantCulture, $"The Vision pipeline published result '{result.ResultId}' but the result NodeId " + $"could not be resolved, so no detection can be selected.")); #pragma warning restore RCS1214 } VisionDetectionSummary? summary = result.DetectionSummary; if (summary is null) { throw new InvalidOperationException(string.Create(CultureInfo.InvariantCulture, $"The Vision result '{result.ResultId}' carries no detection summary.")); } if (summary.TotalDetections <= summary.Items.Count) { return new VisionPickObservation(result, summary.Items, summary.TotalDetections, false); } VisionDetectionResultSnapshot snapshot = await readFullSnapshot(ct).ConfigureAwait(false); var items = new List(snapshot.Detections.Count); for (int i = 0; i < snapshot.Detections.Count; i++) { VisionDetectionDataType detection = snapshot.Detections[i]; items.Add(new VisionDetectionItem { DetectionId = detection.DetectionId ?? string.Empty, ClassLabel = detection.ClassLabel ?? string.Empty, ClassId = detection.ClassId, Confidence = detection.Confidence, HasPose = detection.HasPose, Pose = detection.HasPose ? detection.Pose : null }); } return new VisionPickObservation(result, items.ToArrayOf(), items.Count, true); } /// /// Turns one observation into exactly one submission against an already /// resolved controller. This is the seam the tests drive with a fake /// controller client and a supplied observation. /// internal static async Task SubmitAsync( VisionGuidedPickRequest request, RoboticsResolutionContext context, VisionPickObservation observation, CancellationToken ct) { ValidateRequest(request); ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(observation); (VisionDetectionItem selected, int matched) = SelectDetection(request, observation); VisionPickProvenance provenance = BuildProvenance(observation, selected, matched); string objectClass = string.IsNullOrEmpty(request.ObjectClass) ? selected.ClassLabel : request.ObjectClass; var pick = new PickIntentInput { Source = request.Source, Tool = request.Tool, ObjectClass = objectClass, IntentId = request.PickIntentId, Label = request.Label, BufferMode = request.BufferMode, BlockingMode = request.BlockingMode }; if (string.IsNullOrEmpty(request.Destination)) { IntentDataType intent = RoboticsIntentDtoConverter.ConvertPick(pick, context.Scope); IntentSubmissionResult submission = await context.Client .TrySubmitIntentAsync(intent, ct).ConfigureAwait(false); return new VisionGuidedPickResult { Kind = VisionPickSubmissionKind.Pick, Provenance = provenance, PickSubmission = new VisionPickIntentSubmission { Accepted = submission.Accepted, IntentId = submission.IntentId, Operation = submission.Operation.IsNull ? null : submission.Operation.ToString(), Failure = submission.Failure, Message = submission.Message.Text } }; } var place = new PlaceIntentInput { Destination = request.Destination, Tool = request.Tool, IntentId = request.PlaceIntentId, Label = request.Label, BufferMode = request.BufferMode, BlockingMode = request.BlockingMode }; MissionStepInput[] steps = [ new MissionStepInput { StepId = kPickStepId, Released = true, Intent = new MissionIntentInput { Kind = IntentKind.Pick, Source = pick.Source, Tool = pick.Tool, ObjectClass = pick.ObjectClass, IntentId = pick.IntentId, Label = pick.Label, BufferMode = pick.BufferMode, BlockingMode = pick.BlockingMode } }, new MissionStepInput { StepId = kPlaceStepId, Released = true, Intent = new MissionIntentInput { Kind = IntentKind.Place, Destination = place.Destination, Tool = place.Tool, IntentId = place.IntentId, Label = place.Label, BufferMode = place.BufferMode, BlockingMode = place.BlockingMode } } ]; string missionId = string.IsNullOrEmpty(request.MissionId) ? kMissionIdPrefix + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture) : request.MissionId; uint missionUpdateId = request.MissionUpdateId ?? 1; MissionDataType mission = RoboticsMissionTools.BuildMission( missionId, missionUpdateId, steps, null, request.Label, context.Scope); MissionSubmissionResult missionResult = await context.Client .SubmitMissionAsync(mission, ct).ConfigureAwait(false); var result = new VisionGuidedPickResult { Kind = VisionPickSubmissionKind.Mission, Provenance = provenance, MissionSubmission = new VisionPickMissionSubmission { Accepted = missionResult.Accepted, // The server echoes the MissionId it accepted; an empty echo falls back to the // submitted id exactly as the Robot Intent client SDK does, so the caller can // always address the mission it just created. MissionId = missionResult.MissionId.Length == 0 ? missionId : missionResult.MissionId, MissionUpdateId = missionUpdateId, Operation = missionResult.Operation.IsNull ? null : missionResult.Operation.ToString(), Failure = missionResult.Failure, Message = missionResult.Message.Text, PickStepId = kPickStepId, PlaceStepId = kPlaceStepId } }; if (!missionResult.Accepted || missionResult.Operation.IsNull) { return result; } MissionSnapshot snapshot = await context.Client.Transport .ReadMissionSnapshotAsync(missionResult.Operation, ct).ConfigureAwait(false); result.Steps = MapSteps(snapshot); return result; } /// /// Filters the observed detections and selects one deterministically. /// /// internal static (VisionDetectionItem Selected, int Matched) SelectDetection( VisionGuidedPickRequest request, VisionPickObservation observation) { ArgumentNullException.ThrowIfNull(request); ArgumentNullException.ThrowIfNull(observation); ArrayOf detections = observation.Detections; VisionDetectionItem? best = null; int matched = 0; for (int i = 0; i < detections.Count; i++) { VisionDetectionItem candidate = detections[i]; if (!Matches(request, candidate)) { continue; } matched++; if (best is null || IsBetter(candidate, best)) { best = candidate; } } if (best is null) { throw new InvalidOperationException(string.Create(CultureInfo.InvariantCulture, $"No detection in Vision result '{observation.Result.ResultId}' matched the requested " + $"filters (detectionId='{request.DetectionId ?? ""}', " + $"classLabel='{request.ClassLabel ?? ""}', " + $"minimumConfidence={FormatConfidence(request.MinimumConfidence)}). " + $"The result carries {observation.TotalDetections} detection(s), " + $"{detections.Count} of which were considered.")); } return (best, matched); } /// /// Validates every request field explicitly before any server call. /// /// /// internal static void ValidateRequest(VisionGuidedPickRequest request) { ArgumentNullException.ThrowIfNull(request); RequireText(request.Controller, "controller"); RequireText(request.Pipeline, "pipeline"); RequireText(request.Source, "source"); RequireText(request.Tool, "tool"); RejectBlank(request.Destination, "destination"); RejectBlank(request.DetectionId, "detectionId"); RejectBlank(request.ClassLabel, "classLabel"); RejectBlank(request.ObjectClass, "objectClass"); RejectBlank(request.PickIntentId, "pickIntentId"); RejectBlank(request.PlaceIntentId, "placeIntentId"); RejectBlank(request.Label, "label"); RejectBlank(request.MissionId, "missionId"); if (request.MinimumConfidence.HasValue) { double confidence = request.MinimumConfidence.Value; if (!double.IsFinite(confidence)) { throw new ArgumentException( "'minimumConfidence' must be a finite number.", nameof(request)); } if (confidence is < 0.0 or > 1.0) { throw new ArgumentOutOfRangeException( nameof(request), confidence, "'minimumConfidence' must be between 0 and 1 inclusive."); } } if (!Enum.IsDefined(request.Selection)) { throw new ArgumentOutOfRangeException( nameof(request), request.Selection, "Invalid 'selection' value."); } if (request.BufferMode.HasValue && !Enum.IsDefined(request.BufferMode.Value)) { throw new ArgumentOutOfRangeException( nameof(request), request.BufferMode.Value, "Invalid 'bufferMode' value."); } if (request.BlockingMode.HasValue && !Enum.IsDefined(request.BlockingMode.Value)) { throw new ArgumentOutOfRangeException( nameof(request), request.BlockingMode.Value, "Invalid 'blockingMode' value."); } if (!string.IsNullOrEmpty(request.Destination)) { return; } RejectWithoutDestination(request.PlaceIntentId, "placeIntentId"); RejectWithoutDestination(request.MissionId, "missionId"); if (request.MissionUpdateId.HasValue) { throw new ArgumentException( "'missionUpdateId' requires 'destination'; without it a single Pick intent is submitted.", nameof(request)); } } private static bool Matches(VisionGuidedPickRequest request, VisionDetectionItem candidate) { if (!string.IsNullOrEmpty(request.DetectionId) && !string.Equals(request.DetectionId, candidate.DetectionId, StringComparison.Ordinal)) { return false; } if (!string.IsNullOrEmpty(request.ClassLabel) && !string.Equals(request.ClassLabel, candidate.ClassLabel, StringComparison.Ordinal)) { return false; } return !request.MinimumConfidence.HasValue || candidate.Confidence >= request.MinimumConfidence.Value; } private static bool IsBetter(VisionDetectionItem candidate, VisionDetectionItem best) { // double.CompareTo orders NaN below every number, so a detection whose // confidence the server did not report can never displace a real one. int comparison = candidate.Confidence.CompareTo(best.Confidence); if (comparison != 0) { return comparison > 0; } // Equal confidence: ordinal DetectionId decides, and an identical id keeps // the earlier item, which is the original result order. return string.CompareOrdinal(candidate.DetectionId, best.DetectionId) < 0; } private static VisionPickProvenance BuildProvenance( VisionPickObservation observation, VisionDetectionItem selected, int matched) { VisionInferenceResult result = observation.Result; return new VisionPickProvenance { ResultId = result.ResultId, ResultNodeId = result.ResultNodeId.IsNull ? string.Empty : result.ResultNodeId.ToString(), RequestedPipelineNodeId = result.RequestedPipelineNodeId.IsNull ? string.Empty : result.RequestedPipelineNodeId.ToString(), RequestedPipelineName = result.RequestedPipelineName, PipelineId = result.PipelineId.IsNull ? null : result.PipelineId.ToString(), SensorId = result.SensorId.IsNull ? null : result.SensorId.ToString(), ModelVersionUsed = result.ModelVersionUsed, CreationTime = result.CreationTime == default ? null : result.CreationTime.ToString("o", CultureInfo.InvariantCulture), FrameId = result.FrameId, TotalDetections = observation.TotalDetections, MatchedDetections = matched, FullResultRead = observation.FullResultRead, SelectedDetection = BuildDetection(selected) }; } private static VisionPickDetection BuildDetection(VisionDetectionItem selected) { VisionPose3DDataType? pose = selected.HasPose ? selected.Pose : null; return new VisionPickDetection { DetectionId = selected.DetectionId, ClassLabel = selected.ClassLabel, ClassId = selected.ClassId, Confidence = selected.Confidence, HasPose = selected.HasPose, PoseFrameId = pose?.FrameId, PosePosition = (pose?.Position.ToArray()), PoseOrientation = (pose?.Orientation.ToArray()) }; } private static VisionPickMissionStep[] MapSteps(MissionSnapshot snapshot) { if (snapshot.Steps.Count == 0) { return []; } var steps = new VisionPickMissionStep[snapshot.Steps.Count]; for (int i = 0; i < snapshot.Steps.Count; i++) { MissionStepOperation step = snapshot.Steps[i]; steps[i] = new VisionPickMissionStep { StepId = step.StepId, IntentId = step.IntentId, Operation = step.OperationNodeId.IsNull ? null : step.OperationNodeId.ToString(), State = step.State }; } return steps; } private static string FormatConfidence(double? confidence) { return confidence.HasValue ? confidence.Value.ToString("R", CultureInfo.InvariantCulture) : ""; } private static void RequireText(string? value, string parameterName) { if (string.IsNullOrWhiteSpace(value)) { throw new ArgumentException( string.Create(CultureInfo.InvariantCulture, $"'{parameterName}' is required."), parameterName); } } private static void RejectBlank(string? value, string parameterName) { if (value != null && string.IsNullOrWhiteSpace(value)) { throw new ArgumentException( string.Create(CultureInfo.InvariantCulture, $"'{parameterName}' must be omitted or non-empty."), parameterName); } } private static void RejectWithoutDestination(string? value, string parameterName) { if (!string.IsNullOrEmpty(value)) { // All concatenated operands must remain interpolated for string.Create handler binding. // TODO: Remove when RCS1214 preserves interpolated-string-handler overload binding. #pragma warning disable RCS1214 throw new ArgumentException( string.Create(CultureInfo.InvariantCulture, $"'{parameterName}' requires 'destination'; without it a single Pick intent " + $"is submitted."), parameterName); #pragma warning restore RCS1214 } } private const int kMaxSummaryItems = 100; private const string kPickStepId = "pick"; private const string kPlaceStepId = "place"; private const string kMissionIdPrefix = "vision-pick-"; private readonly OpcUaSessionManager m_sessionManager; } /// /// One bounded perception observation: the inference result, the detections /// selection considers, and whether the full snapshot had to be read. /// internal sealed record VisionPickObservation( VisionInferenceResult Result, ArrayOf Detections, int TotalDetections, bool FullResultRead); }