From e15913cf7e601bcdacdd61eed53b2e908de16655 Mon Sep 17 00:00:00 2001 From: gil zhang Date: Sun, 24 Nov 2024 15:24:16 +0800 Subject: [PATCH] add image --- .../Models/Lingxi/LingxiSpaceType.cs | 2 + .../Functions/CustomGenerateImageFn.cs | 61 +++++++++++++++++++ .../Models/CustomGenerateImageFunctionArgs.cs | 12 ++++ .../Verdure.Braincase.Copilot.Plugin.csproj | 4 ++ .../agent.json | 2 +- .../functions/custom_generate_image.json | 18 ++++++ .../instructions/instruction.liquid | 5 +- .../Controls/LingxiSpace/LingxiSpaceList.xaml | 39 ++++++++---- 8 files changed, 128 insertions(+), 15 deletions(-) create mode 100644 src/Modules/Verdure.Braincase.Copilot.Plugin/Functions/CustomGenerateImageFn.cs create mode 100644 src/Modules/Verdure.Braincase.Copilot.Plugin/Models/CustomGenerateImageFunctionArgs.cs create mode 100644 src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/functions/custom_generate_image.json diff --git a/src/Common/Verdure.Braincase.Core/Models/Lingxi/LingxiSpaceType.cs b/src/Common/Verdure.Braincase.Core/Models/Lingxi/LingxiSpaceType.cs index b9e632c..ba952ca 100644 --- a/src/Common/Verdure.Braincase.Core/Models/Lingxi/LingxiSpaceType.cs +++ b/src/Common/Verdure.Braincase.Core/Models/Lingxi/LingxiSpaceType.cs @@ -11,5 +11,7 @@ public class LingxiSpaceType public const string Word = "word"; + public const string Image = "image"; + public const string Weather = "weather"; } diff --git a/src/Modules/Verdure.Braincase.Copilot.Plugin/Functions/CustomGenerateImageFn.cs b/src/Modules/Verdure.Braincase.Copilot.Plugin/Functions/CustomGenerateImageFn.cs new file mode 100644 index 0000000..2e3da9e --- /dev/null +++ b/src/Modules/Verdure.Braincase.Copilot.Plugin/Functions/CustomGenerateImageFn.cs @@ -0,0 +1,61 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using CommunityToolkit.Mvvm.Messaging; +using Verdure.Braincase.Copilot.Plugin.Models; +using Verdure.Braincase.Core.Contracts.Services; +using Verdure.Braincase.Core.Models.Lingxi; +using Verdure.ElectronBot.Core.Contracts.Services; + +namespace Verdure.Braincase.Copilot.Plugin.Functions; + +public class CustomGenerateImageFn : IFunctionCallback +{ + public string Name => "custom_generate_image"; + + private readonly IServiceProvider _service; + private readonly IBotToolService _botToolService; + private readonly JsonSerializerOptions _options; + private readonly ILingxiSpaceService _lingxiSpaceService; + private readonly IConversationService _conversationService; + public CustomGenerateImageFn(IServiceProvider service, + IBotToolService botToolService, + ILingxiSpaceService lingxiSpaceService, + IConversationService conversationService) + { + _service = service; + _options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + AllowTrailingCommas = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + _botToolService = botToolService; + _lingxiSpaceService = lingxiSpaceService; + _conversationService = conversationService; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "", _options) ?? new CustomGenerateImageFunctionArgs(); + + message.StopCompletion = true; + + var lingxiSpace = await _lingxiSpaceService.AddAsync(new LingxiSpace + { + Id = Guid.NewGuid().ToString(), + ConversationId = _conversationService.ConversationId, + //Content = JsonSerializer.SerializeToDocument(wordContent, _options), + Name = args.ImageName, + Desc = "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/1d/67/20241124/9d34e27a/c2fb7b01-44af-4abd-9929-d520d6c06e39-1.png?Expires=1732515865&OSSAccessKeyId=LTAI5tQZd8AEcZX6KZV4G8qL&Signature=z8%2FYLCgTbFbwDZdVVQapFMgQVzY%3D",//args.ImageDescription, + Type = LingxiSpaceType.Image, + CreatedTime = DateTime.UtcNow + }); + + WeakReferenceMessenger.Default.Send(lingxiSpace); + return true; + } +} diff --git a/src/Modules/Verdure.Braincase.Copilot.Plugin/Models/CustomGenerateImageFunctionArgs.cs b/src/Modules/Verdure.Braincase.Copilot.Plugin/Models/CustomGenerateImageFunctionArgs.cs new file mode 100644 index 0000000..1890856 --- /dev/null +++ b/src/Modules/Verdure.Braincase.Copilot.Plugin/Models/CustomGenerateImageFunctionArgs.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace Verdure.Braincase.Copilot.Plugin.Models; + +public class CustomGenerateImageFunctionArgs +{ + [JsonPropertyName("image_name")] + public string ImageName { get; set; } = string.Empty; + + [JsonPropertyName("image_description")] + public string ImageDescription { get; set; } = string.Empty; +} diff --git a/src/Modules/Verdure.Braincase.Copilot.Plugin/Verdure.Braincase.Copilot.Plugin.csproj b/src/Modules/Verdure.Braincase.Copilot.Plugin/Verdure.Braincase.Copilot.Plugin.csproj index b0d792b..fa4131b 100644 --- a/src/Modules/Verdure.Braincase.Copilot.Plugin/Verdure.Braincase.Copilot.Plugin.csproj +++ b/src/Modules/Verdure.Braincase.Copilot.Plugin/Verdure.Braincase.Copilot.Plugin.csproj @@ -27,6 +27,7 @@ + @@ -52,6 +53,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/agent.json b/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/agent.json index cae95cb..0043a75 100644 --- a/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/agent.json +++ b/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/agent.json @@ -1,6 +1,6 @@ { "name": "工具助手", - "description": "可以控制家电，查询天气，等一些功能。", + "description": "可以控制家电，查询天气，生成图片等一些功能。", "createdDateTime": "2023-07-26T02:29:25.123224Z", "updatedDateTime": "2023-07-26T02:29:25.123274Z", "id": "2e54c75b-03e1-06db-53d9-d86a073675e2", diff --git a/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/functions/custom_generate_image.json b/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/functions/custom_generate_image.json new file mode 100644 index 0000000..32d5599 --- /dev/null +++ b/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/functions/custom_generate_image.json @@ -0,0 +1,18 @@ +{ + "name": "custom_generate_image", + "description": "如果用户想生成图片可以调用此方法进行图片生成。", + "parameters": { + "type": "object", + "properties": { + "image_name": { + "type": "string", + "description": "根据用户描述给图片起个名称。" + }, + "image_description": { + "type": "string", + "description": "用户进行的图片描述。" + } + }, + "required": [ "image_description" ] + } +} \ No newline at end of file diff --git a/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/instructions/instruction.liquid b/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/instructions/instruction.liquid index d2feccc..7b4da06 100644 --- a/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/instructions/instruction.liquid +++ b/src/Modules/Verdure.Braincase.Copilot.Plugin/data/agents/2e54c75b-03e1-06db-53d9-d86a073675e2/instructions/instruction.liquid @@ -1,6 +1,7 @@ -你是一个工具助手，可以帮助用户打开家电，查询天气，等其他的一些工具功能。 +你是一个工具助手，可以帮助用户打开家电，查询天气，生成图片等其他的一些工具功能。 根据一下步骤进行操作: 1: 如果用户一些需求，你需要询问用户是否需要相关帮助。 2: 如果有查天气的需求，你就执行函数 get_weather。 -3: 如果有开灯的需求，你就执行函数 control_the_lights。 +3: 如果有开灯的需求，你就执行函数 control_the_lights。 +4: 如果有生成图片的需求，你就执行函数 custom_generate_image。 diff --git a/src/Modules/Verdure.Braincase.Copilot/Controls/LingxiSpace/LingxiSpaceList.xaml b/src/Modules/Verdure.Braincase.Copilot/Controls/LingxiSpace/LingxiSpaceList.xaml index e4a0706..dcb79bb 100644 --- a/src/Modules/Verdure.Braincase.Copilot/Controls/LingxiSpace/LingxiSpaceList.xaml +++ b/src/Modules/Verdure.Braincase.Copilot/Controls/LingxiSpace/LingxiSpaceList.xaml @@ -29,18 +29,33 @@ - - - - + + + + + + + + + + + + +