From 41ea24b7ebffef141f9236e7505ea01ac7a090cc Mon Sep 17 00:00:00 2001 From: gil zhang Date: Thu, 20 Jul 2023 21:57:35 +0800 Subject: [PATCH] update pose page data --- src/ElectronBot.Braincase/Helpers/EbHelper.cs | 18 +-- .../ViewModels/PoseRecognitionViewModel.cs | 124 ++++++++++++++++-- .../Views/PoseRecognitionPage.xaml | 15 ++- 3 files changed, 136 insertions(+), 21 deletions(-) diff --git a/src/ElectronBot.Braincase/Helpers/EbHelper.cs b/src/ElectronBot.Braincase/Helpers/EbHelper.cs index a4ab52f..4443d80 100644 --- a/src/ElectronBot.Braincase/Helpers/EbHelper.cs +++ b/src/ElectronBot.Braincase/Helpers/EbHelper.cs @@ -267,7 +267,7 @@ public class EbHelper /// /// /// - public static async Task ShowDataToDeviceAsync(SoftwareBitmap? softwareBitmap, EmoticonActionFrame? frameData = null) + public static async Task ShowDataToDeviceAsync(SoftwareBitmap? softwareBitmap, EmoticonActionFrame frameData) { if (softwareBitmap != null) { @@ -294,23 +294,23 @@ public class EbHelper Marshal.Copy(dataMeta, data, 0, 240 * 240 * 3); - var frame = new EmoticonActionFrame(data); + var frame = new EmoticonActionFrame(data, frameData.Enable, + frameData.J1, frameData.J2, frameData.J3, frameData.J4, frameData.J5, frameData.J6); var service = App.GetService(); - ElectronBotHelper.Instance.ModelActionInvoke(new ModelActionFrame(mat2.ToMemoryStream())); + ElectronBotHelper.Instance.ModelActionInvoke( + new ModelActionFrame(mat2.ToMemoryStream(), + false, frameData.J1, frameData.J2, frameData.J3, frameData.J4, frameData.J5, frameData.J6)); await service.SendToUsbDeviceAsync(frame); } else { - if (frameData != null) - { - ElectronBotHelper.Instance.ModelActionInvoke(new ModelActionFrame(new MemoryStream(), false, - frameData.J1, frameData.J2, frameData.J3, frameData.J4, frameData.J5, frameData.J6)); + ElectronBotHelper.Instance.ModelActionInvoke(new ModelActionFrame(new MemoryStream(), false, + frameData.J1, frameData.J2, frameData.J3, frameData.J4, frameData.J5, frameData.J6)); - ElectronBotHelper.Instance.PlayEmoticonActionFrame(frameData); - } + ElectronBotHelper.Instance.PlayEmoticonActionFrame(frameData); } } diff --git a/src/ElectronBot.Braincase/ViewModels/PoseRecognitionViewModel.cs b/src/ElectronBot.Braincase/ViewModels/PoseRecognitionViewModel.cs index 03a9790..d05b986 100644 --- a/src/ElectronBot.Braincase/ViewModels/PoseRecognitionViewModel.cs +++ b/src/ElectronBot.Braincase/ViewModels/PoseRecognitionViewModel.cs @@ -36,6 +36,9 @@ using Windows.Media.Playback; using ElectronBot.Braincase.Extensions; using Vector3 = SharpDX.Vector3; using Verdure.ElectronBot.Core.Models; +using Microsoft.UI.Xaml.Media.Imaging; +using Windows.Media.FaceAnalysis; +using Windows.Media; namespace ElectronBot.Braincase.ViewModels; @@ -138,9 +141,16 @@ public partial class PoseRecognitionViewModel : ObservableRecipient [ObservableProperty] private Image _faceImage = new(); + [ObservableProperty] + private SoftwareBitmapSource _faceBoxSource; + SoftwareBitmap? _frameServerDest = null; + SoftwareBitmap? _faceSoftwareBitmap = null; + private static PoseCpuSolution? calculator; + + private FaceDetector _faceDetector; public Camera Camera { get; @@ -503,6 +513,8 @@ public partial class PoseRecognitionViewModel : ObservableRecipient _isInitialized = true; CameraBackground = new SolidColorBrush(Colors.Green); + + _faceDetector = await FaceDetector.CreateAsync(); } private void Current_SoftwareBitmapFramePosePredictResult(object? sender, PoseOutput e) @@ -547,6 +559,19 @@ public partial class PoseRecognitionViewModel : ObservableRecipient e.PoseLandmarks.Landmark[13].Y * _frameServerDest.PixelHeight)); RightWaveResultLabel = $"RightWave: {rightWaveAngle}"; + var headAngle = AngleHelper.GetPointAngle( + new System.Numerics.Vector2(e.PoseLandmarks.Landmark[11].X * _frameServerDest.PixelWidth, + e.PoseLandmarks.Landmark[11].Y * _frameServerDest.PixelHeight), + new System.Numerics.Vector2(e.PoseLandmarks.Landmark[12].X * _frameServerDest.PixelWidth, + e.PoseLandmarks.Landmark[12].Y * _frameServerDest.PixelHeight), + new System.Numerics.Vector2(e.PoseLandmarks.Landmark[0].X * _frameServerDest.PixelWidth, + e.PoseLandmarks.Landmark[0].Y * _frameServerDest.PixelHeight)); + + if (headAngle > 90) + { + headAngle = 90; + } + var canvasDevice = App.GetService(); if (_canvasImageSource == null) @@ -575,15 +600,17 @@ public partial class PoseRecognitionViewModel : ObservableRecipient var data = new byte[240 * 240 * 3]; - var frame = new EmoticonActionFrame(data, true, 0, (rightWaveAngle / 180) * 30, rightUpAngle, (leftWaveAngle / 180) * 30, leftUpAngle, 0); + var frame = new EmoticonActionFrame(data, true, (headAngle / 90) * 15, (rightWaveAngle / 180) * 30, rightUpAngle, (leftWaveAngle / 180) * 30, leftUpAngle, 0); //待处理面部数据 - await EbHelper.ShowDataToDeviceAsync(null, frame); + await EbHelper.ShowDataToDeviceAsync(_faceSoftwareBitmap, frame); }); } - private void Current_SoftwareBitmapFrameCaptured(object? sender, SoftwareBitmapEventArgs e) + private async void Current_SoftwareBitmapFrameCaptured(object? sender, SoftwareBitmapEventArgs e) { + App.MainWindow.DispatcherQueue.TryEnqueue(async () => + { if (e.SoftwareBitmap is not null) { @@ -597,8 +624,23 @@ public partial class PoseRecognitionViewModel : ObservableRecipient _frameServerDest = e.SoftwareBitmap; + var face = await FaceDetectionAsync(e.SoftwareBitmap); + + if (face is not null) + { + + var source = new SoftwareBitmapSource(); + + await source.SetBitmapAsync(face); + + _faceSoftwareBitmap = face; + // Set the source of the Image control + FaceBoxSource = source; + + } + _ = service.PosePredictResultUnUseQueueAsync(calculator, e.SoftwareBitmap); - } + } }); } public async void UnLoaded() @@ -626,12 +668,14 @@ public partial class PoseRecognitionViewModel : ObservableRecipient { BodyModel.HxTransform3D = _bodyMt * Matrix.RotationY(MathUtil.DegreesToRadians((e.J6))); - //Material = new DiffuseMaterial() - //{ - // EnableUnLit = false, - // DiffuseMap = LoadTextureByStream(e.FrameStream) - //}; - + if (e.FrameStream != null && e.FrameStream.Length > 0) + { + Material = new DiffuseMaterial() + { + EnableUnLit = false, + DiffuseMap = LoadTextureByStream(e.FrameStream) + }; + } var nodeList = HeadModel.GroupNode; @@ -757,4 +801,64 @@ public partial class PoseRecognitionViewModel : ObservableRecipient } } + + private async Task FaceDetectionAsync(SoftwareBitmap? imgSoftwareBitmap) + { + Debug.WriteLine("FrameCaptured"); + Debug.WriteLine($"Frame evaluation started {DateTime.Now}"); + if (imgSoftwareBitmap != null) + { + BitmapPixelFormat bpf = imgSoftwareBitmap.BitmapPixelFormat; + + var uncroppedBitmap = SoftwareBitmap.Convert(imgSoftwareBitmap, BitmapPixelFormat.Nv12); + var faces = await _faceDetector.DetectFacesAsync(uncroppedBitmap); + + if (faces.Count > 0) + { + //crop image to focus on face portion + var faceBox = faces[0].FaceBox; + + VideoFrame inputFrame = VideoFrame.CreateWithSoftwareBitmap(imgSoftwareBitmap); + + VideoFrame tmp = null; + + tmp = new VideoFrame(imgSoftwareBitmap.BitmapPixelFormat, (int)(faceBox.Width + faceBox.Width % 2) - 2, + (int)(faceBox.Height + faceBox.Height % 2) - 2); + + await inputFrame.CopyToAsync(tmp, faceBox, null); + + if (tmp.SoftwareBitmap is not null) + { + return tmp.SoftwareBitmap; + } + } + } + return null; + } + + private static BitmapBounds GetCropBounds(int srcWidth, int srcHeight, int targetWidth, int targetHeight) + { + var modelHeight = targetHeight; + var modelWidth = targetWidth; + BitmapBounds bounds = new BitmapBounds(); + // we need to recalculate the crop bounds in order to correctly center-crop the input image + float flRequiredAspectRatio = (float)modelWidth / modelHeight; + + if (flRequiredAspectRatio * srcHeight > (float)srcWidth) + { + // clip on the y axis + bounds.Height = (uint)Math.Min((srcWidth / flRequiredAspectRatio + 0.5f), srcHeight); + bounds.Width = (uint)srcWidth; + bounds.X = 0; + bounds.Y = (uint)(srcHeight - bounds.Height) / 2; + } + else // clip on the x axis + { + bounds.Width = (uint)Math.Min((flRequiredAspectRatio * srcHeight + 0.5f), srcWidth); + bounds.Height = (uint)srcHeight; + bounds.X = (uint)(srcWidth - bounds.Width) / 2; ; + bounds.Y = 0; + } + return bounds; + } } diff --git a/src/ElectronBot.Braincase/Views/PoseRecognitionPage.xaml b/src/ElectronBot.Braincase/Views/PoseRecognitionPage.xaml index 7bce633..70d5836 100644 --- a/src/ElectronBot.Braincase/Views/PoseRecognitionPage.xaml +++ b/src/ElectronBot.Braincase/Views/PoseRecognitionPage.xaml @@ -46,8 +46,19 @@ + + +