.Net: Add support for Base64 encoded images in MistralAI

dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatCompletion.cs

@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Microsoft.SemanticKernel.Connectors.MistralAI;
+
+namespace ChatCompletion;
+
+// The following example shows how to use Semantic Kernel with MistralAI API
+public class MistralAI_ChatCompletion(ITestOutputHelper output) : BaseTest(output)
+{
+    [Fact]
+    public async Task GetChatMessageContentAsync()
+    {
+        Assert.NotNull(TestConfiguration.MistralAI.ChatModelId);
+        Assert.NotNull(TestConfiguration.MistralAI.ApiKey);
+
+        MistralAIChatCompletionService chatService = new(
+            modelId: TestConfiguration.MistralAI.ChatModelId,
+            apiKey: TestConfiguration.MistralAI.ApiKey);
+
+        var chatHistory = new ChatHistory("You are a librarian, expert about books");
+
+        chatHistory.AddUserMessage("Hi, I'm looking for book suggestions");
+        this.OutputLastMessage(chatHistory);
+
+        var reply = await chatService.GetChatMessageContentAsync(chatHistory, new MistralAIPromptExecutionSettings { MaxTokens = 200 });
+        Console.WriteLine(reply);
+    }
+
+    [Fact]
+    public async Task GetChatMessageContentUsingImageContentAsync()
+    {
+        Assert.NotNull(TestConfiguration.MistralAI.ImageModelId);
+        Assert.NotNull(TestConfiguration.MistralAI.ApiKey);
+
+        // Create a logging handler to output HTTP requests and responses
+        var handler = new LoggingHandler(new HttpClientHandler(), this.Output);
+        var httpClient = new HttpClient(handler);
+
+        MistralAIChatCompletionService chatService = new(
+            modelId: TestConfiguration.MistralAI.ImageModelId,
+            apiKey: TestConfiguration.MistralAI.ApiKey,
+            httpClient: httpClient);
+
+        var chatHistory = new ChatHistory();
+
+        var chatMessage = new ChatMessageContent(AuthorRole.User, "What's in this image?");
+        chatMessage.Items.Add(new ImageContent("data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEA2ADYAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAQABADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5rooor8DP9oD/2Q=="));
+
+        chatHistory.Add(chatMessage);
+        this.OutputLastMessage(chatHistory);
+
+        var reply = await chatService.GetChatMessageContentAsync(chatHistory, new MistralAIPromptExecutionSettings { MaxTokens = 200 });
+        Console.WriteLine(reply);
+    }
+}

dotnet/samples/Concepts/README.md

@@ -76,6 +76,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
 - [OpenAI_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_FunctionCalling.cs)
 - [OpenAI_ReasonedFunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ReasonedFunctionCalling.cs)
 - [MultipleProviders_ChatHistoryReducer](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MuiltipleProviders_ChatHistoryReducer.cs)
+- [MistralAI_ChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatCompletion.cs)
 - [MistralAI_ChatPrompt](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatPrompt.cs)
 - [MistralAI_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_FunctionCalling.cs)
 - [MistralAI_StreamingFunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_StreamingFunctionCalling.cs)

dotnet/src/Connectors/Connectors.MistralAI/Client/ImageUrlChunk.cs

@@ -1,11 +1,10 @@
 // Copyright (c) Microsoft. All rights reserved.
 
-using System;
 using System.Text.Json.Serialization;
 
 namespace Microsoft.SemanticKernel.Connectors.MistralAI.Client;
-internal class ImageUrlChunk(Uri imageUrl) : ContentChunk(ContentChunkType.ImageUrl)
+internal class ImageUrlChunk(string imageUrl) : ContentChunk(ContentChunkType.ImageUrl)
 {
     [JsonPropertyName("image_url")]
-    public string ImageUrl { get; set; } = imageUrl.ToString();
+    public string ImageUrl { get; set; } = imageUrl;
 }

dotnet/src/Connectors/Connectors.MistralAI/Client/MistralClient.cs

@@ -784,9 +784,16 @@ internal List<MistralChatMessage> ToMistralChatMessages(ChatMessageContent chatM
             {
                 content.Add(new TextChunk(textContent.Text!));
             }
-            else if (item is ImageContent imageContent && imageContent.Uri is not null)
+            else if (item is ImageContent imageContent)
             {
-                content.Add(new ImageUrlChunk(imageContent.Uri));
+                if (imageContent.Uri is not null)
+                {
+                    content.Add(new ImageUrlChunk(imageContent.Uri.ToString()));
+                }
+                else if (imageContent.DataUri is not null)
+                {
+                    content.Add(new ImageUrlChunk(imageContent.DataUri));
+                }
             }
             else
             {

dotnet/src/IntegrationTests/Connectors/MistralAI/ChatCompletion/MistralAIChatCompletionTests.cs

@@ -147,6 +147,28 @@ public async Task ValidateGetChatMessageContentsWithImageAsync()
         Assert.Contains("Snow", response[0].Content, System.StringComparison.InvariantCultureIgnoreCase);
     }
 
+    [Fact(Skip = "This test is for manual verification.")]
+    public async Task ValidateGetChatMessageContentsWithImageDataUriAsync()
+    {
+        // Arrange
+        var model = this._configuration["MistralAI:ImageModelId"];
+        var apiKey = this._configuration["MistralAI:ApiKey"];
+        var service = new MistralAIChatCompletionService(model!, apiKey!, httpClient: this._httpClient);
+
+        // Act
+        var chatHistory = new ChatHistory
+        {
+            new ChatMessageContent(AuthorRole.User, "What's in this image?"),
+            new ChatMessageContent(AuthorRole.User, [new ImageContent("data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEA2ADYAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAQABADASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5rooor8DP9oD/2Q==")])
+        };
+        var response = await service.GetChatMessageContentsAsync(chatHistory, this._executionSettings);
+
+        // Assert
+        Assert.NotNull(response);
+        Assert.Single(response);
+        Assert.Contains("square", response[0].Content, System.StringComparison.InvariantCultureIgnoreCase);
+    }
+
     [Fact(Skip = "This test is for manual verification.")]
     public async Task ValidateGetChatMessageContentsWithImageAndJsonFormatAsync()
     {

dotnet/src/InternalUtilities/samples/InternalUtilities/TestConfiguration.cs

@@ -213,6 +213,7 @@ public class MistralAIConfig
         public string ApiKey { get; set; }
         public string ChatModelId { get; set; }
         public string EmbeddingModelId { get; set; }
+        public string ImageModelId { get; set; }
     }
 
     public class GoogleAIConfig