com.openai.unity 5.2.1 (#120)

- Removed dynamic types in favor of object to fix crashing on il2cpp - Added additional Preserve attributes and constructors to stripped types - Changed tts clip download cache location - Updated chat samples package with tts voice of assistant - Added ability to reference a Texture2D object for vision requests - Updated docs
RageAgainstThePixel · Nov 10, 2023 · 6ea846e · 6ea846e
1 parent ec4f620
commit 6ea846e
Show file tree

Hide file tree

Showing 20 changed files with 405 additions and 122 deletions.
diff --git a/OpenAI/Packages/com.openai.unity/Documentation~/README.md b/OpenAI/Packages/com.openai.unity/Documentation~/README.md
@@ -62,31 +62,32 @@ The recommended installation method is though the unity package manager and [Ope
 - [Chat](#chat)
   - [Chat Completions](#chat-completions)
   - [Streaming](#chat-streaming)
-  - [Functions](#chat-functions)
+  - [Tools](#chat-tools) :new:
+  - [Vision](#chat-vision) :new:
 - [Edits](#edits)
   - [Create Edit](#create-edit)
 - [Embeddings](#embeddings)
   - [Create Embedding](#create-embeddings)
-- [Audio](#audio) :construction:
-  - [Create Speech](#create-speech) :new:
+- [Audio](#audio)
+  - [Create Speech](#create-speech)
   - [Create Transcription](#create-transcription)
   - [Create Translation](#create-translation)
-- [Images](#images) :construction:
-  - [Create Image](#create-image) :new:
-  - [Edit Image](#edit-image) :new:
-  - [Create Image Variation](#create-image-variation) :new:
+- [Images](#images)
+  - [Create Image](#create-image)
+  - [Edit Image](#edit-image)
+  - [Create Image Variation](#create-image-variation)
 - [Files](#files)
   - [List Files](#list-files)
   - [Upload File](#upload-file)
   - [Delete File](#delete-file)
   - [Retrieve File Info](#retrieve-file-info)
   - [Download File Content](#download-file-content)
-- [Fine Tuning](#fine-tuning) :construction:
-  - [Create Fine Tune Job](#create-fine-tune-job) :new:
-  - [List Fine Tune Jobs](#list-fine-tune-jobs) :new:
-  - [Retrieve Fine Tune Job Info](#retrieve-fine-tune-job-info) :new:
-  - [Cancel Fine Tune Job](#cancel-fine-tune-job) :new:
-  - [List Fine Tune Job Events](#list-fine-tune-job-events) :new:
+- [Fine Tuning](#fine-tuning)
+  - [Create Fine Tune Job](#create-fine-tune-job)
+  - [List Fine Tune Jobs](#list-fine-tune-jobs)
+  - [Retrieve Fine Tune Job Info](#retrieve-fine-tune-job-info)
+  - [Cancel Fine Tune Job](#cancel-fine-tune-job)
+  - [List Fine Tune Job Events](#list-fine-tune-job-events)
 - [Moderations](#moderations)
   - [Create Moderation](#create-moderation)
 
@@ -99,6 +100,8 @@ There are 4 ways to provide your API keys, in order of precedence:
 3. [Load key from configuration file](#load-key-from-configuration-file)
 4. [Use System Environment Variables](#use-system-environment-variables)
 
+You use the `OpenAIAuthentication` when you initialize the API as shown:
+
 #### Pass keys directly with constructor
 
 :warning: We recommended using the environment variables to load the API key instead of having it hard coded in your source. It is not recommended use this method in production, but only for accepting user credentials, local testing and quick start scenarios.
@@ -408,7 +411,7 @@ await api.ChatEndpoint.StreamCompletionAsync(chatRequest, result =>
 });
 ```
 
-##### [Chat Functions](https://platform.openai.com/docs/api-reference/chat/create#chat/create-functions)
+##### [Chat Tools](https://platform.openai.com/docs/guides/function-calling)
 
 > Only available with the latest 0613 model series!
 
@@ -425,8 +428,8 @@ foreach (var message in messages)
     Debug.Log($"{message.Role}: {message.Content}");
 }
 
-// Define the functions that the assistant is able to use:
-var functions = new List<Function>
+// Define the tools that the assistant is able to use:
+var tools = new List<Tool>
 {
     new Function(
         nameof(WeatherService.GetCurrentWeather),
@@ -451,45 +454,88 @@ var functions = new List<Function>
             })
 };
 
-var chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo");
+var chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto");
 var result = await api.ChatEndpoint.GetCompletionAsync(chatRequest);
 messages.Add(result.FirstChoice.Message);
+
 Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}");
+
 var locationMessage = new Message(Role.User, "I'm in Glasgow, Scotland");
 messages.Add(locationMessage);
 Debug.Log($"{locationMessage.Role}: {locationMessage.Content}");
-chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo");
+chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto");
 result = await api.ChatEndpoint.GetCompletionAsync(chatRequest);
+
 messages.Add(result.FirstChoice.Message);
 
 if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content))
 {
-    // It's possible that the assistant will also ask you which units you want the temperature in.
     Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}");
 
     var unitMessage = new Message(Role.User, "celsius");
     messages.Add(unitMessage);
     Debug.Log($"{unitMessage.Role}: {unitMessage.Content}");
-    chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo");
+    chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto");
     result = await api.ChatEndpoint.GetCompletionAsync(chatRequest);
 }
 
-Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Function.Name} | Finish Reason: {result.FirstChoice.FinishReason}");
-Debug.Log($"{result.FirstChoice.Message.Function.Arguments}");
-var functionArgs = JsonConvert.DeserializeObject<WeatherArgs>(result.FirstChoice.Message.Function.Arguments.ToString());
+var usedTool = result.FirstChoice.Message.ToolCalls[0];
+Debug.Log($"{result.FirstChoice.Message.Role}: {usedTool.Function.Name} | Finish Reason: {result.FirstChoice.FinishReason}");
+Debug.Log($"{usedTool.Function.Arguments}");
+var functionArgs = JsonSerializer.Deserialize<WeatherArgs>(usedTool.Function.Arguments.ToString());
 var functionResult = WeatherService.GetCurrentWeather(functionArgs);
-messages.Add(new Message(Role.Function, functionResult, nameof(WeatherService.GetCurrentWeather)));
-Debug.Log($"{Role.Function}: {functionResult}");
+messages.Add(new Message(usedTool, functionResult));
+Debug.Log($"{Role.Tool}: {functionResult}");
 // System: You are a helpful weather assistant.
 // User: What's the weather like today?
 // Assistant: Sure, may I know your current location? | Finish Reason: stop
 // User: I'm in Glasgow, Scotland
-// Assistant: GetCurrentWeather | Finish Reason: function_call
+// Assistant: GetCurrentWeather | Finish Reason: tool_calls
 // {
 //   "location": "Glasgow, Scotland",
 //   "unit": "celsius"
 // }
-// Function: The current weather in Glasgow, Scotland is 20 celsius
+// Tool: The current weather in Glasgow, Scotland is 20 celsius
+```
+
+##### [Chat Vision](https://platform.openai.com/docs/guides/vision)
+
+:construction: This feature is in beta!
+
+> Currently, GPT-4 with vision does not support the message.name parameter, functions/tools, nor the response_format parameter.
+
+```csharp
+var api = new OpenAIClient();
+var messages = new List<Message>
+{
+    new Message(Role.System, "You are a helpful assistant."),
+    new Message(Role.User, new List<Content>
+    {
+        new Content(ContentType.Text, "What's in this image?"),
+        new Content(ContentType.ImageUrl, "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")
+    })
+};
+var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview");
+var result = await apiChatEndpoint.GetCompletionAsync(chatRequest);
+Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}");
+```
+
+You can even pass in a `Texture2D`!
+
+```csharp
+var api = new OpenAIClient();
+var messages = new List<Message>
+{
+    new Message(Role.System, "You are a helpful assistant."),
+    new Message(Role.User, new List<Content>
+    {
+        new Content(ContentType.Text, "What's in this image?"),
+        new Content(texture)
+    })
+};
+var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview");
+var result = await apiChatEndpoint.GetCompletionAsync(chatRequest);
+Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}");
 ```
 
 ### [Edits](https://platform.openai.com/docs/api-reference/edits)

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs
@@ -1,8 +1,8 @@
 // Licensed under the MIT License. See LICENSE in the project root for license information.
 
-using System;
 using Newtonsoft.Json;
 using OpenAI.Extensions;
+using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
@@ -61,11 +61,7 @@ public async Task<Tuple<string, AudioClip>> CreateSpeechAsync(SpeechRequest requ
             var response = await Rest.PostAsync(GetUrl("/speech"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken);
             response.Validate(EnableDebug);
             await Rest.ValidateCacheDirectoryAsync();
-            var cacheDirectory = Rest.DownloadCacheDirectory
-                  .CreateNewDirectory(nameof(OpenAI)
-                  .CreateNewDirectory(nameof(Audio)
-                  .CreateNewDirectory("Speech")));
-            var cachedPath = Path.Combine(cacheDirectory, $"{DateTime.UtcNow:yyyyMMddThhmmss}.{ext}");
+            var cachedPath = Path.Combine(Rest.DownloadCacheDirectory, $"{request.Voice}-{DateTime.UtcNow:yyyyMMddThhmmss}.{ext}");
             await File.WriteAllBytesAsync(cachedPath, response.Data, cancellationToken).ConfigureAwait(true);
             var clip = await Rest.DownloadAudioClipAsync($"file://{cachedPath}", audioFormat, cancellationToken: cancellationToken);
             return new Tuple<string, AudioClip>(cachedPath, clip);

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechRequest.cs
@@ -6,6 +6,7 @@
 
 namespace OpenAI.Audio
 {
+    [Preserve]
     public sealed class SpeechRequest
     {
         /// <summary>
@@ -16,6 +17,7 @@ public sealed class SpeechRequest
         /// <param name="voice">The voice to use when generating the audio.</param>
         /// <param name="responseFormat">The format to audio in. Supported formats are mp3, opus, aac, and flac.</param>
         /// <param name="speed">The speed of the generated audio. Select a value from 0.25 to 4.0. 1.0 is the default.</param>
+        [Preserve]
         public SpeechRequest(string input, Model model = null, SpeechVoice voice = SpeechVoice.Alloy, SpeechResponseFormat responseFormat = SpeechResponseFormat.MP3, float? speed = null)
         {
             Input = input;

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatRequest.cs
@@ -13,6 +13,7 @@ namespace OpenAI.Chat
     public sealed class ChatRequest
     {
         /// <inheritdoc />
+        [Preserve]
         [Obsolete("Use new constructor arguments")]
         public ChatRequest(
             IEnumerable<Message> messages,
@@ -354,7 +355,7 @@ public ChatRequest(
         /// </summary>
         [Preserve]
         [JsonProperty("tool_choice")]
-        public dynamic ToolChoice { get; }
+        public object ToolChoice { get; }
 
         /// <summary>
         /// A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.
@@ -370,7 +371,7 @@ public ChatRequest(
         [Preserve]
         [Obsolete("Use ToolChoice")]
         [JsonProperty("function_call")]
-        public dynamic FunctionCall { get; }
+        public object FunctionCall { get; }
 
         /// <summary>
         /// An optional list of functions to get arguments for.
@@ -381,6 +382,7 @@ public ChatRequest(
         public IReadOnlyList<Function> Functions { get; }
 
         /// <inheritdoc />
+        [Preserve]
         public override string ToString() => JsonConvert.SerializeObject(this, OpenAIClient.JsonSerializationOptions);
     }
 }
diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponseFormat.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponseFormat.cs
@@ -1,3 +1,5 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using System.Runtime.Serialization;
 
 namespace OpenAI.Chat

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Content.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Content.cs
@@ -1,11 +1,31 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using Newtonsoft.Json;
+using System;
+using UnityEngine;
 using UnityEngine.Scripting;
 
 namespace OpenAI.Chat
 {
     [Preserve]
     public sealed class Content
     {
+        [Preserve]
+        public static implicit operator Content(string content) => new Content(content);
+
+        [Preserve]
+        public Content(string text)
+            : this(ContentType.Text, text)
+        {
+        }
+
+        [Preserve]
+        public Content(Texture2D texture)
+            : this(ContentType.ImageUrl, $"data:image/jpeg;base64,{Convert.ToBase64String(texture.EncodeToPNG())}")
+        {
+        }
+
+        [Preserve]
         public Content(ContentType type, string input)
         {
             Type = type;

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ContentType.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ContentType.cs
@@ -1,3 +1,5 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using System.Runtime.Serialization;
 
 namespace OpenAI.Chat

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Conversation.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Conversation.cs
@@ -28,10 +28,13 @@ public Conversation([JsonProperty("messages")] List<Message> messages)
         /// Appends <see cref="Message"/> to the end of <see cref="Messages"/>.
         /// </summary>
         /// <param name="message">The message to add to the <see cref="Conversation"/>.</param>
+        [Preserve]
         public void AppendMessage(Message message) => messages.Add(message);
 
+        [Preserve]
         public override string ToString() => JsonConvert.SerializeObject(this, OpenAIClient.JsonSerializationOptions);
 
+        [Preserve]
         public static implicit operator string(Conversation conversation) => conversation.ToString();
     }
 }
diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/FinishDetails.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/FinishDetails.cs
@@ -1,3 +1,5 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using Newtonsoft.Json;
 using UnityEngine.Scripting;
 
@@ -6,6 +8,9 @@ namespace OpenAI.Chat
     [Preserve]
     public sealed class FinishDetails
     {
+        [Preserve]
+        public FinishDetails() { }
+
         [Preserve]
         [JsonProperty("type")]
         public string Type { get; private set; }

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ImageUrl.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ImageUrl.cs
@@ -1,3 +1,5 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using Newtonsoft.Json;
 using UnityEngine.Scripting;
 

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Message.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Message.cs
@@ -114,26 +114,25 @@ public Role Role
             private set => role = value;
         }
 
-        [Preserve]
         [SerializeField]
         [TextArea(1, 30)]
         private string content;
 
-        private dynamic contentList;
+        private object contentList;
 
         /// <summary>
         /// The contents of the message.
         /// </summary>
         [Preserve]
         [JsonProperty("content", DefaultValueHandling = DefaultValueHandling.Populate, NullValueHandling = NullValueHandling.Include, Required = Required.AllowNull)]
-        public dynamic Content
+        public object Content
         {
             get => contentList ?? content;
             private set
             {
-                if (value is string)
+                if (value is string s)
                 {
-                    content = value;
+                    content = s;
                 }
                 else
                 {

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ResponseFormat.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ResponseFormat.cs
@@ -1,3 +1,5 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using Newtonsoft.Json;
 using UnityEngine.Scripting;
 

diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Tool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Tool.cs
@@ -1,14 +1,20 @@
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
 using Newtonsoft.Json;
 using UnityEngine.Scripting;
 
 namespace OpenAI.Chat
 {
+    [Preserve]
     public sealed class Tool
     {
+        [Preserve]
         public Tool() { }
 
+        [Preserve]
         public Tool(Tool other) => CopyFrom(other);
 
+        [Preserve]
         public Tool(Function function)
         {
             Function = function;
@@ -31,8 +37,10 @@ public Tool(Function function)
         [JsonProperty("function")]
         public Function Function { get; private set; }
 
+        [Preserve]
         public static implicit operator Tool(Function function) => new Tool(function);
 
+        [Preserve]
         internal void CopyFrom(Tool other)
         {
             if (!string.IsNullOrWhiteSpace(other?.Id))