.Net: feat/declarative agents (#9849)

fixes #9848
This is a draft PR, at this point it's not ready for a full review.
I'd like feedback on the location of the new extension method, and on
the fact that I had to add a reference to the agents project.
Here is the rationale for placing it here: it needs access to the
copilot agent plugins extension method and a few other types.
Here is the rationale for adding the reference to the agents project: it
needs access to the kernel agent type.

---------

Signed-off-by: Vincent Biret <[email protected]>
Co-authored-by: Chris <[email protected]>

dotnet/samples/Concepts/Agents/DeclarativeAgents.cs

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft. All rights reserved.
+using System.Text;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Agents;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Plugins;
+
+namespace Agents;
+
+public class DeclarativeAgents(ITestOutputHelper output) : BaseAgentsTest(output)
+{
+    [InlineData("SchedulingAssistant.json", "Read the body of my last five emails, if any contain a meeting request for today, check that it's already on my calendar, if not, call out which email it is.")]
+    [Theory]
+    public async Task LoadsAgentFromDeclarativeAgentManifestAsync(string agentFileName, string input)
+    {
+        var kernel = CreateKernel();
+        kernel.AutoFunctionInvocationFilters.Add(new ExpectedSchemaFunctionFilter());
+        var manifestLookupDirectory = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "Resources", "DeclarativeAgents");
+        var manifestFilePath = Path.Combine(manifestLookupDirectory, agentFileName);
+
+        var parameters = await CopilotAgentBasedPlugins.GetAuthenticationParametersAsync();
+
+        var agent = await kernel.CreateChatCompletionAgentFromDeclarativeAgentManifestAsync<ChatCompletionAgent>(manifestFilePath, parameters);
+
+        Assert.NotNull(agent);
+        Assert.NotNull(agent.Name);
+        Assert.NotEmpty(agent.Name);
+        Assert.NotNull(agent.Description);
+        Assert.NotEmpty(agent.Description);
+        Assert.NotNull(agent.Instructions);
+        Assert.NotEmpty(agent.Instructions);
+
+        ChatMessageContent message = new(AuthorRole.User, input);
+        ChatHistory chatHistory = [message];
+        StringBuilder sb = new();
+        await foreach (ChatMessageContent response in agent.InvokeAsync(chatHistory))
+        {
+            chatHistory.Add(response);
+            sb.Append(response.Content);
+        }
+        Assert.NotEmpty(chatHistory.Skip(1));
+    }
+    private Kernel CreateKernel()
+    {
+        IKernelBuilder builder = Kernel.CreateBuilder();
+
+        base.AddChatCompletionToKernel(builder);
+
+        return builder.Build();
+    }
+    private sealed class ExpectedSchemaFunctionFilter : IAutoFunctionInvocationFilter
+    {//TODO: this eventually needs to be added to all CAP or DA but we're still discussing where should those facilitators live
+        public async Task OnAutoFunctionInvocationAsync(AutoFunctionInvocationContext context, Func<AutoFunctionInvocationContext, Task> next)
+        {
+            await next(context);
+
+            if (context.Result.ValueType == typeof(RestApiOperationResponse))
+            {
+                var openApiResponse = context.Result.GetValue<RestApiOperationResponse>();
+                if (openApiResponse?.ExpectedSchema is not null)
+                {
+                    openApiResponse.ExpectedSchema = null;
+                }
+            }
+        }
+    }
+}

dotnet/samples/Concepts/Concepts.csproj

@@ -131,6 +131,9 @@
     <Content Include="Resources\Plugins\RepairServicePlugin\repair-service.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
+    <Content Include="Resources\DeclarativeAgents\*">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
   <ItemGroup>
     <EmbeddedResource Include="Resources\*">

dotnet/samples/Concepts/Plugins/CopilotAgentBasedPlugins.cs

@@ -87,9 +87,8 @@ private void WriteSampleHeadingToConsole(string pluginToTest, string functionToT
         Console.WriteLine($"======== Calling Plugin Function: {pluginToTest}.{functionToTest} with parameters {arguments?.Select(x => x.Key + " = " + x.Value).Aggregate((x, y) => x + ", " + y)} ========");
         Console.WriteLine();
     }
-    private async Task AddCopilotAgentPluginsAsync(Kernel kernel, params string[] pluginNames)
+    internal static async Task<CopilotAgentPluginParameters> GetAuthenticationParametersAsync()
     {
-#pragma warning disable SKEXP0050
         if (TestConfiguration.MSGraph.Scopes is null)
         {
             throw new InvalidOperationException("Missing Scopes configuration for Microsoft Graph API.");
@@ -131,6 +130,12 @@ private async Task AddCopilotAgentPluginsAsync(Kernel kernel, params string[] pl
                 { "https://api.nasa.gov/planetary", nasaOpenApiFunctionExecutionParameters }
             }
         };
+        return apiManifestPluginParameters;
+    }
+    private async Task AddCopilotAgentPluginsAsync(Kernel kernel, params string[] pluginNames)
+    {
+#pragma warning disable SKEXP0050
+        var apiManifestPluginParameters = await GetAuthenticationParametersAsync().ConfigureAwait(false);
         var manifestLookupDirectory = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "Resources", "Plugins", "CopilotAgentPlugins");
 
         foreach (var pluginName in pluginNames)

dotnet/samples/Concepts/Resources/DeclarativeAgents/SchedulingAssistant.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "https://aka.ms/json-schemas/copilot/declarative-agent/v1.0/schema.json",
+  "version": "v1.0",
+  "instructions": "$[file('scheduling-assistant-instructions.txt')]",
+  "name": "SchedulingAssistant",
+  "description": "This agent helps you schedule meetings and send messages.",
+  "actions": [
+    {
+      "id": "CalendarPlugin",
+      "file": "../Plugins/CopilotAgentPlugins/CalendarPlugin/calendar-apiplugin.json"
+    },
+    {
+      "id": "MessagesPlugin",
+      "file": "../Plugins/CopilotAgentPlugins/MessagesPlugin/messages-apiplugin.json"
+    }
+  ]
+}

dotnet/samples/Concepts/Resources/DeclarativeAgents/scheduling-assistant-instructions.txt

@@ -0,0 +1 @@
+You are a personal assistant to the user. You help recap the last received emails, the upcoming meetings, and reply to any emails upon user's request. You can only use the calendar and messages plugins. Whenever you make HTTP REST request, you MUST select the fewest fields you need from the API to ensure a great user experience. If you need to select the body field, you MUST select the bodyPreview field instead.

dotnet/src/Agents/Abstractions/Agent.cs

@@ -27,9 +27,9 @@ public abstract class Agent
     /// <summary>
     /// The identifier of the agent (optional).
     /// </summary>
-    /// <reamarks>
+    /// <remarks>
     /// Default to a random guid value, but may be overridden.
-    /// </reamarks>
+    /// </remarks>
     public string Id { get; init; } = Guid.NewGuid().ToString();
 
     /// <summary>

dotnet/src/Functions/Functions.OpenApi.Extensions/Extensions/CopilotAgentPluginKernelExtensions.cs

@@ -79,12 +79,12 @@ public static async Task<KernelPlugin> CreatePluginFromCopilotAgentPluginAsync(
 
         var results = await PluginManifestDocument.LoadAsync(CopilotAgentFileJsonContents, new ReaderOptions
         {
-            ValidationRules = new() // Disable validation rules
+            ValidationRules = [] // Disable validation rules
         }).ConfigureAwait(false);
 
         if (!results.IsValid)
         {
-            var messages = results.Problems.Select(p => p.Message).Aggregate((a, b) => $"{a}, {b}");
+            var messages = results.Problems.Select(static p => p.Message).Aggregate(static (a, b) => $"{a}, {b}");
             throw new InvalidOperationException($"Error loading the manifest: {messages}");
         }
 

dotnet/src/Functions/Functions.OpenApi.Extensions/Extensions/DeclarativeAgentExtensions.cs

@@ -0,0 +1,131 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Plugins.Manifest;
+using Microsoft.SemanticKernel.Agents;
+using Microsoft.SemanticKernel.Plugins.OpenApi;
+using Microsoft.SemanticKernel.Plugins.OpenApi.Extensions;
+
+namespace Microsoft.SemanticKernel;
+
+/// <summary>
+/// Provides extension methods for loading and managing declarative agents and their Copilot Agent Plugins.
+/// </summary>
+public static class DeclarativeAgentExtensions
+{
+    /// <summary>
+    /// Creates a chat completion agent from a declarative agent manifest asynchronously.
+    /// </summary>
+    /// <typeparam name="T">The type of the agent to create.</typeparam>
+    /// <param name="kernel">The kernel instance.</param>
+    /// <param name="filePath">The file path of the declarative agent manifest.</param>
+    /// <param name="pluginParameters">Optional parameters for the Copilot Agent Plugin setup.</param>
+    /// <param name="promptExecutionSettings">Optional prompt execution settings. Ensure you enable function calling.</param>
+    /// <param name="cancellationToken">Optional cancellation token.</param>
+    /// <returns>A task that represents the asynchronous operation. The task result contains the created chat completion agent.</returns>
+    public static async Task<T> CreateChatCompletionAgentFromDeclarativeAgentManifestAsync<T>(
+        this Kernel kernel,
+        string filePath,
+        CopilotAgentPluginParameters? pluginParameters = null,
+        PromptExecutionSettings? promptExecutionSettings = default,
+        CancellationToken cancellationToken = default)
+        where T : KernelAgent, new()
+    {
+        Verify.NotNull(kernel);
+        Verify.NotNullOrWhiteSpace(filePath);
+
+        var loggerFactory = kernel.LoggerFactory;
+        var logger = loggerFactory.CreateLogger(typeof(DeclarativeAgentExtensions)) ?? NullLogger.Instance;
+        using var declarativeAgentFileJsonContents = DocumentLoader.LoadDocumentFromFilePathAsStream(filePath,
+            logger);
+
+        var results = await DCManifestDocument.LoadAsync(declarativeAgentFileJsonContents, new ReaderOptions
+        {
+            ValidationRules = [] // Disable validation rules
+        }).ConfigureAwait(false);
+
+        if (!results.IsValid)
+        {
+            var messages = results.Problems.Select(static p => p.Message).Aggregate(static (a, b) => $"{a}, {b}");
+            throw new InvalidOperationException($"Error loading the manifest: {messages}");
+        }
+
+        var document = results.Document ?? throw new InvalidOperationException("Error loading the manifest");
+        var manifestDirectory = Path.GetDirectoryName(filePath);
+        document.Instructions = await GetEffectiveInstructionsAsync(manifestDirectory, document.Instructions, logger, cancellationToken).ConfigureAwait(false);
+
+        var agent = new T
+        {
+            Name = document.Name,
+            Instructions = document.Instructions,
+            Kernel = kernel,
+            Arguments = new KernelArguments(promptExecutionSettings ?? new PromptExecutionSettings()
+            {
+                FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(),
+            }),
+            Description = document.Description,
+            LoggerFactory = loggerFactory,
+            Id = string.IsNullOrEmpty(document.Id) ? Guid.NewGuid().ToString() : document.Id!,
+        };
+
+        if (document.Capabilities is { Count: > 0 })
+        {
+            logger.LogWarning("Importing capabilities from declarative agent is not supported in semantic kernel.");
+        }
+
+        if (document.Actions is { Count: > 0 })
+        {
+            logger.LogInformation("Importing {ActionsCount} actions from declarative agent.", document.Actions.Count);
+            await Task.WhenAll(document.Actions.Select(action => ImportCAPFromActionAsync(action, manifestDirectory, kernel, pluginParameters, logger, cancellationToken))).ConfigureAwait(false);
+        }
+        return agent;
+    }
+    private static async Task ImportCAPFromActionAsync(DCAction action, string? manifestDirectory, Kernel kernel, CopilotAgentPluginParameters? pluginParameters, ILogger logger, CancellationToken cancellationToken)
+    {
+        try
+        {
+            var capManifestPath = GetFullPath(manifestDirectory, action.File);
+            logger.LogInformation("Importing action {ActionName} from declarative agent from path {Path}.", action.Id, capManifestPath);
+            await kernel.ImportPluginFromCopilotAgentPluginAsync(action.Id, capManifestPath, pluginParameters, cancellationToken).ConfigureAwait(false);
+        }
+        catch (Exception ex) when (ex is FileNotFoundException or InvalidOperationException)
+        {
+            logger.LogError(ex, "Error importing action {ActionName} from declarative agent.", action.Id);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Error importing action {ActionName} from declarative agent.", action.Id);
+            throw;
+        }
+    }
+    private static async Task<string?> GetEffectiveInstructionsAsync(string? manifestFilePath, string? source, ILogger logger, CancellationToken cancellationToken)
+    {
+        if (string.IsNullOrEmpty(source) ||
+            !source!.StartsWith("$[file('", StringComparison.OrdinalIgnoreCase) ||
+            !source.EndsWith("')]", StringComparison.OrdinalIgnoreCase))
+        {
+            return source;
+        }
+#if NETCOREAPP3_0_OR_GREATER
+        var filePath = source[8..^3];
+#else
+        var filePath = source.Substring(8, source.Length - 11);
+#endif
+        filePath = GetFullPath(manifestFilePath, filePath);
+        return await DocumentLoader.LoadDocumentFromFilePathAsync(filePath, logger, cancellationToken).ConfigureAwait(false);
+    }
+    private static string GetFullPath(string? manifestDirectory, string relativeOrAbsolutePath)
+    {
+        return !Path.IsPathRooted(relativeOrAbsolutePath) && !relativeOrAbsolutePath.StartsWith("http", StringComparison.OrdinalIgnoreCase)
+            ? string.IsNullOrEmpty(manifestDirectory)
+                ? throw new InvalidOperationException("Invalid manifest file path.")
+                : Path.Combine(manifestDirectory, relativeOrAbsolutePath)
+            : relativeOrAbsolutePath;
+    }
+}

dotnet/src/Functions/Functions.OpenApi.Extensions/Functions.OpenApi.Extensions.csproj

@@ -20,5 +20,6 @@
   <ItemGroup>
     <ProjectReference Include="..\..\SemanticKernel.Core\SemanticKernel.Core.csproj" />
     <ProjectReference Include="..\Functions.OpenApi\Functions.OpenApi.csproj" />
+    <ProjectReference Include="..\..\Agents\Abstractions\Agents.Abstractions.csproj"/>
   </ItemGroup>
 </Project>