/* ========================================================================
* Copyright (c) 2005-2026 The OPC Foundation, Inc. All rights reserved.
*
* OPC Foundation MIT License 1.00
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
* The complete license agreement can be found here:
* http://opcfoundation.org/License/MIT/1.00/
* ======================================================================*/
using System;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
#if BINPICKING_CLIENT_MCP
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
#endif
using Microsoft.Extensions.Logging;
#if BINPICKING_CLIENT_MCP
using Microsoft.Extensions.Logging.Console;
using ModelContextProtocol.Server;
#endif
using Opc.Ua;
using Opc.Ua.Client;
#if BINPICKING_CLIENT_MCP
using Opc.Ua.Mcp;
#endif
using Opc.Ua.OpenUsd.Client;
using Opc.Ua.Robotics.Client;
using Opc.Ua.Robotics.Client.Intent;
namespace BinPickingClient
{
///
/// Bin-picking client entry point. Connects to the bin-picking cell server, exposes
/// the composed Vision + Robotics MCP catalogue for an external agent, optionally
/// opens the OpenUSD viewport so a human can watch the arm move, and can run the
/// scripted pick-and-place demonstration end to end.
///
internal static partial class Program
{
[STAThread]
public static async Task Main(string[] args)
{
BinPickingClientOptions options = BinPickingClientOptions.Parse(args);
if (options.Transport is not null &&
!BinPickingClientOptions.TryParseMcpTransport(options.Transport, out _))
{
Console.Error.WriteLine(
$"Unknown MCP transport '{options.Transport}'. Valid transports: stdio, http, sse.");
return 2;
}
BinPickingClientMcpTransportSelection mcpTransport = options.SelectMcpTransport();
if (options.Mcp)
{
#if BINPICKING_CLIENT_MCP
Console.Error.WriteLine(mcpTransport.Message);
#else
Console.Error.WriteLine(
"MCP hosting is unavailable for this target framework. Run the sample without --mcp, " +
"or use the net8.0, net9.0, or net10.0 target framework for MCP hosting.");
return 2;
#endif
}
// A console provider, or the sample's own logging goes nowhere: the OpenUSD
// connector reports what it bound and any target it had to leave unresolved,
// and a live stream that silently binds nothing looks exactly like one that
// works. Errors go to stderr so stdout stays clean for MCP stdio transport.
LogLevel minimumLevel = options.Verbose ? LogLevel.Debug : LogLevel.Information;
using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder
.SetMinimumLevel(minimumLevel)
.AddFilter(level => level >= minimumLevel)
.AddConsole(console => console.LogToStandardErrorThreshold = LogLevel.Trace));
ILogger logger = loggerFactory.CreateLogger("BinPickingClient");
ITelemetryContext telemetry = DefaultTelemetry.Create(builder => builder
.SetMinimumLevel(minimumLevel)
.AddFilter(level => level >= minimumLevel)
.AddConsole(console => console.LogToStandardErrorThreshold = LogLevel.Trace));
using CancellationTokenSource lifetime = options.Seconds > 0
? new CancellationTokenSource(TimeSpan.FromSeconds(options.Seconds))
: new CancellationTokenSource();
BinPickingSampleSession sample = await BinPickingSampleSession.ConnectAsync(
options, telemetry, lifetime.Token).ConfigureAwait(false);
await using (sample.ConfigureAwait(false))
{
LogConnected(logger, options.ServerUrl);
RobotIntentClient intentClient = sample.Session.RobotIntent(telemetry, sample.Streaming);
ArrayOf controllers =
await intentClient.DiscoverControllersAsync(lifetime.Token).ConfigureAwait(false);
if (controllers.Count == 0)
{
Console.Error.WriteLine(
"No Robot Intent controllers were advertised at the conformant Server/RobotIntent/Controllers path.");
return 2;
}
RobotIntentControllerClient controller = intentClient.Controller(controllers[0].NodeId);
RobotIntentControllerInfo controllerInfo = await controller
.ReadAsync(lifetime.Token).ConfigureAwait(false);
string controllerName = string.IsNullOrEmpty(controllerInfo.BrowseName.Name)
? controllers[0].BrowseName.Name ?? "(unnamed)"
: controllerInfo.BrowseName.Name;
Console.Error.WriteLine(
$"Controller: {controllerName} ({controllerInfo.NodeId})");
bool commandGranted = false;
CommandAuthorityLease? authority = null;
#if BINPICKING_CLIENT_MCP
IHost? mcpHost = null;
#endif
// Only take command authority when this process is going to command the
// robot: the scripted demo, or an MCP host driving the cell through this
// session. A viewer is an observer, and taking an exclusive lease just to
// watch locks out the very agent the sample exists to serve - an agent on
// its own MCP session gets every intent refused while a window is open.
bool willCommand = options.Demo || options.Mcp;
if (willCommand)
{
try
{
authority = await controller.RequestAuthorityAsync(lifetime.Token).ConfigureAwait(false);
if (authority.Granted)
{
Console.Error.WriteLine("Command authority: granted for this session.");
commandGranted = true;
}
else
{
Console.Error.WriteLine(
$"Command authority: held by {authority.CurrentOwner}; submissions may be refused.");
}
}
catch (ServiceResultException exception)
when (exception.StatusCode == StatusCodes.BadUserAccessDenied)
{
Console.Error.WriteLine(
"Command authority request was denied: the connecting identity lacks the Operator role. " +
"Continuing in read-only mode so vision inference and MCP tool discovery remain visible.");
}
}
else
{
Console.Error.WriteLine(
"Command authority: not requested - this session only observes, so another client " +
"or an agent can command the cell while the viewport is open.");
}
int exitCode = 0;
try
{
#if BINPICKING_CLIENT_MCP
if (options.Mcp)
{
mcpHost = await StartMcpHostAsync(
mcpTransport, options, sample.Session, logger, lifetime.Token).ConfigureAwait(false);
}
#endif
if (options.Demo && options.View)
{
// Order matters: the scripted loop is over in seconds, so running it
// before the viewport opens leaves nothing to watch - the arm has already
// parked by the time the window appears. Open the viewport, wait for the
// live stream to be subscribed, and only then command the robot.
var streamReady = new TaskCompletionSource(
TaskCreationOptions.RunContinuationsAsynchronously);
Task viewport = RunViewportIfAvailableAsync(
sample, options, telemetry, streamReady, lifetime.Token);
await WaitForLiveStreamAsync(streamReady, viewport, lifetime.Token).ConfigureAwait(false);
var runner = new BinPickingDemoRunner(sample, telemetry, logger, options);
exitCode = await runner.RunAsync(
controller, controllerInfo, commandGranted, lifetime.Token).ConfigureAwait(false);
Console.Error.WriteLine(
"Scripted loop finished; the viewport stays open so the cell can still be " +
"inspected. Close the window to exit.");
_ = await viewport.ConfigureAwait(false);
}
else if (options.Demo)
{
var runner = new BinPickingDemoRunner(sample, telemetry, logger, options);
exitCode = await runner.RunAsync(
controller, controllerInfo, commandGranted, lifetime.Token).ConfigureAwait(false);
}
else if (options.View)
{
bool closedByUser = await RunViewportIfAvailableAsync(
sample, options, telemetry, null, lifetime.Token).ConfigureAwait(false);
#if BINPICKING_CLIENT_MCP
if (!closedByUser && options.Mcp && !options.Demo)
{
// The viewer never opened, or it failed. An agent is still driving the
// cell over MCP, so keep serving instead of exiting underneath it.
Console.Error.WriteLine(
"MCP server still running without a viewport; connect an MCP client to drive " +
"the cell. Press Ctrl+C to exit.");
await WaitForMcpServerAsync(mcpHost!, lifetime.Token).ConfigureAwait(false);
}
#endif
}
else if (options.Mcp && !options.Demo)
{
#if BINPICKING_CLIENT_MCP
Console.Error.WriteLine(
"MCP server running; connect an MCP client to drive the cell. Press Ctrl+C to exit.");
await WaitForMcpServerAsync(mcpHost!, lifetime.Token).ConfigureAwait(false);
#endif
}
else if (!options.Mcp && !options.Demo && !options.View)
{
Console.Error.WriteLine(
"Nothing to do: no --mcp, --demo, or --view supplied. Connected and read the controller " +
"capabilities to prove the session is healthy; exiting.");
}
}
finally
{
#if BINPICKING_CLIENT_MCP
if (mcpHost is not null)
{
await StopMcpHostAsync(mcpHost).ConfigureAwait(false);
}
#endif
if (authority is not null)
{
await authority.DisposeAsync().ConfigureAwait(false);
}
}
return exitCode;
}
}
///
/// Waits until the live OpenUSD stream is subscribed, so a caller can command motion
/// that the viewport will actually show. Gives up if the viewport ends first (it is
/// optional and may be unavailable) or after a short grace period, because a demo that
/// cannot be watched is still better than one that never runs.
///
private static async Task WaitForLiveStreamAsync(
TaskCompletionSource streamReady,
Task viewport,
CancellationToken cancellationToken)
{
Task completed = await Task.WhenAny(
streamReady.Task,
viewport,
Task.Delay(TimeSpan.FromSeconds(30), cancellationToken)).ConfigureAwait(false);
if (completed == streamReady.Task)
{
// Let the first subscription values land before commanding, so the opening
// frame shows the cell at rest rather than mid-motion.
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken).ConfigureAwait(false);
}
}
private static async Task RunViewportIfAvailableAsync(
BinPickingSampleSession sample,
BinPickingClientOptions options,
ITelemetryContext telemetry,
TaskCompletionSource? streamReady,
CancellationToken cancellationToken)
{
if (!UsdViewHostLoader.TryLoad(out IUsdViewHost? viewHost, out string reason))
{
Console.Error.WriteLine(
"Viewport unavailable; the sample continues without a viewer. " + reason);
streamReady?.TrySetResult(false);
return false;
}
string liveLayerPath = PrepareLiveLayerPath(options);
string cacheDir = options.FetchAssetsDirectory
?? Path.GetDirectoryName(liveLayerPath)
?? AppContext.BaseDirectory;
string stagePath = Path.Combine(cacheDir, "stage.usda");
// The viewport needs the served geometry: without it only the live override layer
// composes, which carries transforms but no geometry and renders as an empty scene.
await FetchAssetsAsync(sample.Session, cacheDir, cancellationToken).ConfigureAwait(false);
if (!File.Exists(stagePath))
{
Console.Error.WriteLine(
"No fetched stage.usda exists; viewport will open the live override layer only.");
stagePath = liveLayerPath;
}
var viewOptions = new UsdViewOptions
{
StagePath = stagePath,
Renderer = options.Renderer,
// Defaults to the stage's fixed observer camera, which shows the cell
// working; --camera auto hands framing back to the viewer. Do not point
// this at /World/Robot/.../Camera by default - that is the eye-in-hand
// sensor on the flange, so it shows what the tool sees, not the cell.
CameraPath = options.CameraPath,
Title = "OPC UA Bin-picking Viewer",
Telemetry = telemetry
};
Console.Error.WriteLine("Opening OpenUSD viewport for the bin-picking cell.");
Console.Error.WriteLine(options.CameraPath is { Length: > 0 } cameraPath
? $"Opening on the stage camera {cameraPath}."
: "No stage camera requested; the viewport frames the scene itself.");
try
{
await RunViewportOnStaThreadAsync(
viewHost!, viewOptions, sample.Session, streamReady, cancellationToken)
.ConfigureAwait(false);
return true;
}
#pragma warning disable CA1031 // The viewer is a third-party UI; no exception from it should end the session.
catch (Exception exception) when (!cancellationToken.IsCancellationRequested)
#pragma warning restore CA1031
{
// The viewport is optional. A failure inside the renderer or its window code must
// not take down an MCP session an agent is driving, so report it and carry on.
Console.Error.WriteLine(
"The OpenUSD viewport ended with an error; the sample continues without a viewer. " +
exception.Message);
return false;
}
}
private static Task RunViewportOnStaThreadAsync(
IUsdViewHost viewHost,
UsdViewOptions viewOptions,
ISession session,
TaskCompletionSource? streamReady,
CancellationToken cancellationToken)
{
var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var uiThread = new Thread(() =>
{
try
{
viewHost.RunViewport(
viewOptions,
async (sink, ct) => await StreamOpenUsdAsync(
session, sink, viewOptions.Telemetry, streamReady, ct)
.ConfigureAwait(false),
cancellationToken);
completion.TrySetResult(true);
}
#pragma warning disable CA1031
catch (Exception exception)
#pragma warning restore CA1031
{
completion.TrySetException(exception);
}
})
{
IsBackground = false,
Name = "Bin-picking OpenUSD viewport"
};
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
uiThread.SetApartmentState(ApartmentState.STA);
}
uiThread.Start();
return completion.Task;
}
private static async Task StreamOpenUsdAsync(
ISession session,
IUsdSink sink,
ITelemetryContext? telemetry,
TaskCompletionSource? streamReady,
CancellationToken cancellationToken)
{
// Thread telemetry in: without it the connector logs to NullLogger, so a live
// stream that binds nothing, or that leaves every target unresolved, looks
// exactly like one that is working.
var connector = new OpenUsdConnector(
session, sink, new OpenUsdConnectorOptions { EnableCommands = false }, telemetry);
await using (connector.ConfigureAwait(false))
{
await connector.StartAsync(cancellationToken).ConfigureAwait(false);
Console.Error.WriteLine("Live OpenUSD stream started; the viewport now follows the cell.");
streamReady?.TrySetResult(true);
try
{
await PumpWhileViewportIsOpenAsync(session, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Viewport shutdown ends the live stream.
}
finally
{
Console.Error.WriteLine("Live OpenUSD stream stopping.");
await connector.StopAsync(CancellationToken.None).ConfigureAwait(false);
}
}
}
///
/// Keeps the client doing work for as long as the viewport is open.
///
///
/// This used to be a single infinite delay, which is the natural thing to write:
/// the connector's subscriptions push updates, so the host has nothing else to do.
/// It also stops the viewport updating. Measured against this cell, an idle host
/// applies no stage updates at all while a busy one applies them normally
/// (openusd-dotnet issue 17), so a viewer that only watches shows a frozen scene
/// while the robot is demonstrably moving - which is exactly the case when an
/// agent, rather than this client, is driving the cell. Reading the controller on
/// a timer keeps the host doing something between updates. The read is a real
/// one so this is a poll rather than a spin, and it is cheap next to rendering.
///
private static async Task PumpWhileViewportIsOpenAsync(
ISession session,
CancellationToken cancellationToken)
{
var serverStatus = new ReadValueId
{
NodeId = global::Opc.Ua.VariableIds.Server_ServerStatus_CurrentTime,
AttributeId = Attributes.Value
};
ArrayOf nodesToRead = [serverStatus];
while (!cancellationToken.IsCancellationRequested)
{
try
{
_ = await session.ReadAsync(
null, 0, TimestampsToReturn.Neither, nodesToRead, cancellationToken)
.ConfigureAwait(false);
}
#pragma warning disable CA1031 // A read failure must not close a viewport the user is watching.
catch (Exception) when (!cancellationToken.IsCancellationRequested)
#pragma warning restore CA1031
{
// A dropped read is the reconnect handler's business, not the viewer's.
}
await Task.Delay(ViewportPumpInterval, cancellationToken).ConfigureAwait(false);
}
}
private static async Task FetchAssetsAsync(
ISession session, string cacheDir, CancellationToken cancellationToken)
{
Directory.CreateDirectory(cacheDir);
var fetcher = new OpenUsdConnector(session, new MockUsdSink(), enableCommands: false);
await using (fetcher.ConfigureAwait(false))
{
System.Collections.Generic.List fetched =
await fetcher.FetchServedAssetsAsync(cacheDir, cancellationToken).ConfigureAwait(false);
if (fetched.Count == 0)
{
Console.Error.WriteLine(
"Server did not advertise served OpenUSD assets; viewport will use the live override layer only.");
return;
}
Console.Error.WriteLine(
$"Fetched {fetched.Count} OpenUSD asset(s) into {cacheDir}.");
}
}
private static string PrepareLiveLayerPath(BinPickingClientOptions options)
{
string root = options.FetchAssetsDirectory ?? Path.Combine(GetPrivateStateRoot(), "bin-picking");
Directory.CreateDirectory(root);
string path = Path.Combine(root, "live.usda");
if (!File.Exists(path))
{
File.WriteAllText(path, "#usda 1.0\n(\n doc = \"OPC UA -> OpenUSD live override layer\"\n)\n");
}
return path;
}
private static string GetPrivateStateRoot()
{
string baseDirectory = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
if (string.IsNullOrEmpty(baseDirectory))
{
baseDirectory = AppContext.BaseDirectory;
}
string root = Path.Combine(baseDirectory, "OPC Foundation", "BinPickingClient");
Directory.CreateDirectory(root);
return root;
}
#if BINPICKING_CLIENT_MCP
private static async Task StartMcpHostAsync(
BinPickingClientMcpTransportSelection transport,
BinPickingClientOptions options,
ISession session,
ILogger logger,
CancellationToken cancellationToken)
{
IHost host = transport.Transport == BinPickingClientMcpTransport.Stdio
? BuildStdioMcpHost()
: BuildHttpMcpHost(options.Port);
OpcUaSessionManager sessionManager = host.Services.GetRequiredService();
await sessionManager.RegisterExistingSessionAsync(
"bin-picking", session, "Anonymous", cancellationToken).ConfigureAwait(false);
int toolCount = host.Services.GetServices().Count();
LogMcpCatalogueSize(logger, toolCount);
Console.Error.WriteLine(
$"MCP catalogue exposes {toolCount} tools (Vision + Robotics + Connection).");
await host.StartAsync(cancellationToken).ConfigureAwait(false);
string transportName = transport.Transport.ToOptionValue();
LogMcpHostStarted(logger, transportName);
if (transport.Transport == BinPickingClientMcpTransport.Http)
{
Console.Error.WriteLine(
$"MCP server is listening on http://localhost:{options.Port}/mcp with Vision + Robotics tools.");
}
else
{
Console.Error.WriteLine("MCP server is listening on stdio with Vision + Robotics tools.");
}
return host;
}
private static IHost BuildStdioMcpHost()
{
HostApplicationBuilder builder = Host.CreateApplicationBuilder();
ConfigureMcpLogging(builder.Logging, useStdioTransport: true);
ConfigureMcpServices(builder.Services);
IMcpServerBuilder mcpServerBuilder = builder.Services
.AddMcpServer()
.WithStdioServerTransport();
ConfigureVisionAndRoboticsTools(mcpServerBuilder);
return builder.Build();
}
private static WebApplication BuildHttpMcpHost(int port)
{
WebApplicationBuilder builder = WebApplication.CreateBuilder();
ConfigureMcpLogging(builder.Logging, useStdioTransport: false);
ConfigureMcpServices(builder.Services);
IMcpServerBuilder mcpServerBuilder = builder.Services
.AddMcpServer()
.WithHttpTransport();
ConfigureVisionAndRoboticsTools(mcpServerBuilder);
WebApplication app = builder.Build();
app.MapMcp("/mcp");
app.Urls.Add($"http://localhost:{port}");
return app;
}
private static void ConfigureMcpServices(IServiceCollection services)
{
McpToolProfileSet profiles = new McpToolProfileSet(McpToolProfile.Vision).With(McpToolProfile.Robotics);
services.AddOpcUaMcpCore(new OpcUaMcpOptions { ToolProfiles = profiles });
services.AddOpcUaMcpVision();
services.AddOpcUaMcpRobotics();
}
private static void ConfigureVisionAndRoboticsTools(IMcpServerBuilder mcpServerBuilder)
{
McpToolProfileSet profiles = new McpToolProfileSet(McpToolProfile.Vision).With(McpToolProfile.Robotics);
mcpServerBuilder
.WithOpcUaMcpFilters()
.WithOpcUaCoreTools(profiles)
.WithOpcUaVisionTools(profiles)
.WithOpcUaRoboticsTools(profiles);
}
private static void ConfigureMcpLogging(ILoggingBuilder logging, bool useStdioTransport)
{
logging.ClearProviders();
logging.SetMinimumLevel(LogLevel.Information);
logging.AddSimpleConsole(options =>
{
options.UseUtcTimestamp = true;
options.TimestampFormat = "yyyy-MM-dd HH:mm:ss ";
});
logging.Services.Configure(o =>
o.LogToStandardErrorThreshold = useStdioTransport ? LogLevel.Trace : LogLevel.Error);
}
private static async Task WaitForMcpServerAsync(IHost host, CancellationToken cancellationToken)
{
try
{
await host.WaitForShutdownAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
// Ctrl+C or --seconds is the expected way for the host to stop.
}
}
private static async Task StopMcpHostAsync(IHost host)
{
try
{
await host.StopAsync(CancellationToken.None).ConfigureAwait(false);
}
finally
{
host.Dispose();
}
}
#endif
[LoggerMessage(EventId = BinPickingClientEventIds.Connected, Level = LogLevel.Information,
Message = "Connected to bin-picking cell at {ServerUrl}.")]
private static partial void LogConnected(ILogger logger, string serverUrl);
#if BINPICKING_CLIENT_MCP
[LoggerMessage(EventId = BinPickingClientEventIds.McpCatalogueSize, Level = LogLevel.Information,
Message = "MCP catalogue registered with {ToolCount} tools.")]
private static partial void LogMcpCatalogueSize(ILogger logger, int toolCount);
[LoggerMessage(EventId = BinPickingClientEventIds.McpHostStarted, Level = LogLevel.Information,
Message = "MCP host started with transport {Transport}.")]
private static partial void LogMcpHostStarted(ILogger logger, string transport);
#endif
///
/// How often a watching client reads the server while the viewport is open. Fast
/// enough that the host is never idle for long, slow enough to stay a poll.
///
private static readonly TimeSpan ViewportPumpInterval = TimeSpan.FromMilliseconds(50);
}
}