From 6ea846ebc789ec19356ca43daca4aa66280ca3ca Mon Sep 17 00:00:00 2001 From: Stephen Hodgson Date: Fri, 10 Nov 2023 00:03:57 -0500 Subject: [PATCH] com.openai.unity 5.2.1 (#120) - Removed dynamic types in favor of object to fix crashing on il2cpp - Added additional Preserve attributes and constructors to stripped types - Changed tts clip download cache location - Updated chat samples package with tts voice of assistant - Added ability to reference a Texture2D object for vision requests - Updated docs --- .../com.openai.unity/Documentation~/README.md | 100 ++++++++---- .../Runtime/Audio/AudioEndpoint.cs | 8 +- .../Runtime/Audio/SpeechRequest.cs | 2 + .../Runtime/Chat/ChatRequest.cs | 6 +- .../Runtime/Chat/ChatResponseFormat.cs | 2 + .../com.openai.unity/Runtime/Chat/Content.cs | 20 +++ .../Runtime/Chat/ContentType.cs | 2 + .../Runtime/Chat/Conversation.cs | 3 + .../Runtime/Chat/FinishDetails.cs | 5 + .../com.openai.unity/Runtime/Chat/ImageUrl.cs | 2 + .../com.openai.unity/Runtime/Chat/Message.cs | 9 +- .../Runtime/Chat/ResponseFormat.cs | 2 + .../com.openai.unity/Runtime/Chat/Tool.cs | 8 + .../Runtime/Extensions/StringExtensions.cs | 8 +- .../Samples~/Chat/ChatBehaviour.cs | 46 ++++-- .../Samples~/Chat/OpenAIChatSample.unity | 147 +++++++++++++++--- .../Tests/TestFixture_03_Chat.cs | 43 ++++- OpenAI/Packages/com.openai.unity/package.json | 2 +- OpenAI/ProjectSettings/ProjectSettings.asset | 12 +- README.md | 100 ++++++++---- 20 files changed, 405 insertions(+), 122 deletions(-) diff --git a/OpenAI/Packages/com.openai.unity/Documentation~/README.md b/OpenAI/Packages/com.openai.unity/Documentation~/README.md index 572b49eb..473c767a 100644 --- a/OpenAI/Packages/com.openai.unity/Documentation~/README.md +++ b/OpenAI/Packages/com.openai.unity/Documentation~/README.md @@ -62,31 +62,32 @@ The recommended installation method is though the unity package manager and [Ope - [Chat](#chat) - [Chat Completions](#chat-completions) - [Streaming](#chat-streaming) - - [Functions](#chat-functions) + - [Tools](#chat-tools) :new: + - [Vision](#chat-vision) :new: - [Edits](#edits) - [Create Edit](#create-edit) - [Embeddings](#embeddings) - [Create Embedding](#create-embeddings) -- [Audio](#audio) :construction: - - [Create Speech](#create-speech) :new: +- [Audio](#audio) + - [Create Speech](#create-speech) - [Create Transcription](#create-transcription) - [Create Translation](#create-translation) -- [Images](#images) :construction: - - [Create Image](#create-image) :new: - - [Edit Image](#edit-image) :new: - - [Create Image Variation](#create-image-variation) :new: +- [Images](#images) + - [Create Image](#create-image) + - [Edit Image](#edit-image) + - [Create Image Variation](#create-image-variation) - [Files](#files) - [List Files](#list-files) - [Upload File](#upload-file) - [Delete File](#delete-file) - [Retrieve File Info](#retrieve-file-info) - [Download File Content](#download-file-content) -- [Fine Tuning](#fine-tuning) :construction: - - [Create Fine Tune Job](#create-fine-tune-job) :new: - - [List Fine Tune Jobs](#list-fine-tune-jobs) :new: - - [Retrieve Fine Tune Job Info](#retrieve-fine-tune-job-info) :new: - - [Cancel Fine Tune Job](#cancel-fine-tune-job) :new: - - [List Fine Tune Job Events](#list-fine-tune-job-events) :new: +- [Fine Tuning](#fine-tuning) + - [Create Fine Tune Job](#create-fine-tune-job) + - [List Fine Tune Jobs](#list-fine-tune-jobs) + - [Retrieve Fine Tune Job Info](#retrieve-fine-tune-job-info) + - [Cancel Fine Tune Job](#cancel-fine-tune-job) + - [List Fine Tune Job Events](#list-fine-tune-job-events) - [Moderations](#moderations) - [Create Moderation](#create-moderation) @@ -99,6 +100,8 @@ There are 4 ways to provide your API keys, in order of precedence: 3. [Load key from configuration file](#load-key-from-configuration-file) 4. [Use System Environment Variables](#use-system-environment-variables) +You use the `OpenAIAuthentication` when you initialize the API as shown: + #### Pass keys directly with constructor :warning: We recommended using the environment variables to load the API key instead of having it hard coded in your source. It is not recommended use this method in production, but only for accepting user credentials, local testing and quick start scenarios. @@ -408,7 +411,7 @@ await api.ChatEndpoint.StreamCompletionAsync(chatRequest, result => }); ``` -##### [Chat Functions](https://platform.openai.com/docs/api-reference/chat/create#chat/create-functions) +##### [Chat Tools](https://platform.openai.com/docs/guides/function-calling) > Only available with the latest 0613 model series! @@ -425,8 +428,8 @@ foreach (var message in messages) Debug.Log($"{message.Role}: {message.Content}"); } -// Define the functions that the assistant is able to use: -var functions = new List +// Define the tools that the assistant is able to use: +var tools = new List { new Function( nameof(WeatherService.GetCurrentWeather), @@ -451,45 +454,88 @@ var functions = new List }) }; -var chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo"); +var chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto"); var result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); messages.Add(result.FirstChoice.Message); + Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); + var locationMessage = new Message(Role.User, "I'm in Glasgow, Scotland"); messages.Add(locationMessage); Debug.Log($"{locationMessage.Role}: {locationMessage.Content}"); -chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo"); +chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto"); result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); + messages.Add(result.FirstChoice.Message); if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content)) { - // It's possible that the assistant will also ask you which units you want the temperature in. Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); var unitMessage = new Message(Role.User, "celsius"); messages.Add(unitMessage); Debug.Log($"{unitMessage.Role}: {unitMessage.Content}"); - chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo"); + chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto"); result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); } -Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Function.Name} | Finish Reason: {result.FirstChoice.FinishReason}"); -Debug.Log($"{result.FirstChoice.Message.Function.Arguments}"); -var functionArgs = JsonConvert.DeserializeObject(result.FirstChoice.Message.Function.Arguments.ToString()); +var usedTool = result.FirstChoice.Message.ToolCalls[0]; +Debug.Log($"{result.FirstChoice.Message.Role}: {usedTool.Function.Name} | Finish Reason: {result.FirstChoice.FinishReason}"); +Debug.Log($"{usedTool.Function.Arguments}"); +var functionArgs = JsonSerializer.Deserialize(usedTool.Function.Arguments.ToString()); var functionResult = WeatherService.GetCurrentWeather(functionArgs); -messages.Add(new Message(Role.Function, functionResult, nameof(WeatherService.GetCurrentWeather))); -Debug.Log($"{Role.Function}: {functionResult}"); +messages.Add(new Message(usedTool, functionResult)); +Debug.Log($"{Role.Tool}: {functionResult}"); // System: You are a helpful weather assistant. // User: What's the weather like today? // Assistant: Sure, may I know your current location? | Finish Reason: stop // User: I'm in Glasgow, Scotland -// Assistant: GetCurrentWeather | Finish Reason: function_call +// Assistant: GetCurrentWeather | Finish Reason: tool_calls // { // "location": "Glasgow, Scotland", // "unit": "celsius" // } -// Function: The current weather in Glasgow, Scotland is 20 celsius +// Tool: The current weather in Glasgow, Scotland is 20 celsius +``` + +##### [Chat Vision](https://platform.openai.com/docs/guides/vision) + +:construction: This feature is in beta! + +> Currently, GPT-4 with vision does not support the message.name parameter, functions/tools, nor the response_format parameter. + +```csharp +var api = new OpenAIClient(); +var messages = new List +{ + new Message(Role.System, "You are a helpful assistant."), + new Message(Role.User, new List + { + new Content(ContentType.Text, "What's in this image?"), + new Content(ContentType.ImageUrl, "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg") + }) +}; +var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview"); +var result = await apiChatEndpoint.GetCompletionAsync(chatRequest); +Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}"); +``` + +You can even pass in a `Texture2D`! + +```csharp +var api = new OpenAIClient(); +var messages = new List +{ + new Message(Role.System, "You are a helpful assistant."), + new Message(Role.User, new List + { + new Content(ContentType.Text, "What's in this image?"), + new Content(texture) + }) +}; +var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview"); +var result = await apiChatEndpoint.GetCompletionAsync(chatRequest); +Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}"); ``` ### [Edits](https://platform.openai.com/docs/api-reference/edits) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs b/OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs index 5d66703b..38d75956 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Audio/AudioEndpoint.cs @@ -1,8 +1,8 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. -using System; using Newtonsoft.Json; using OpenAI.Extensions; +using System; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -61,11 +61,7 @@ public async Task> CreateSpeechAsync(SpeechRequest requ var response = await Rest.PostAsync(GetUrl("/speech"), payload, new RestParameters(client.DefaultRequestHeaders), cancellationToken); response.Validate(EnableDebug); await Rest.ValidateCacheDirectoryAsync(); - var cacheDirectory = Rest.DownloadCacheDirectory - .CreateNewDirectory(nameof(OpenAI) - .CreateNewDirectory(nameof(Audio) - .CreateNewDirectory("Speech"))); - var cachedPath = Path.Combine(cacheDirectory, $"{DateTime.UtcNow:yyyyMMddThhmmss}.{ext}"); + var cachedPath = Path.Combine(Rest.DownloadCacheDirectory, $"{request.Voice}-{DateTime.UtcNow:yyyyMMddThhmmss}.{ext}"); await File.WriteAllBytesAsync(cachedPath, response.Data, cancellationToken).ConfigureAwait(true); var clip = await Rest.DownloadAudioClipAsync($"file://{cachedPath}", audioFormat, cancellationToken: cancellationToken); return new Tuple(cachedPath, clip); diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechRequest.cs index 0b3fcbf4..3bd71c3a 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechRequest.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Audio/SpeechRequest.cs @@ -6,6 +6,7 @@ namespace OpenAI.Audio { + [Preserve] public sealed class SpeechRequest { /// @@ -16,6 +17,7 @@ public sealed class SpeechRequest /// The voice to use when generating the audio. /// The format to audio in. Supported formats are mp3, opus, aac, and flac. /// The speed of the generated audio. Select a value from 0.25 to 4.0. 1.0 is the default. + [Preserve] public SpeechRequest(string input, Model model = null, SpeechVoice voice = SpeechVoice.Alloy, SpeechResponseFormat responseFormat = SpeechResponseFormat.MP3, float? speed = null) { Input = input; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatRequest.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatRequest.cs index a0a6caa8..8c29a1f7 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatRequest.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatRequest.cs @@ -13,6 +13,7 @@ namespace OpenAI.Chat public sealed class ChatRequest { /// + [Preserve] [Obsolete("Use new constructor arguments")] public ChatRequest( IEnumerable messages, @@ -354,7 +355,7 @@ public ChatRequest( /// [Preserve] [JsonProperty("tool_choice")] - public dynamic ToolChoice { get; } + public object ToolChoice { get; } /// /// A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. @@ -370,7 +371,7 @@ public ChatRequest( [Preserve] [Obsolete("Use ToolChoice")] [JsonProperty("function_call")] - public dynamic FunctionCall { get; } + public object FunctionCall { get; } /// /// An optional list of functions to get arguments for. @@ -381,6 +382,7 @@ public ChatRequest( public IReadOnlyList Functions { get; } /// + [Preserve] public override string ToString() => JsonConvert.SerializeObject(this, OpenAIClient.JsonSerializationOptions); } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponseFormat.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponseFormat.cs index 18a9b95f..28ab467c 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponseFormat.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ChatResponseFormat.cs @@ -1,3 +1,5 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using System.Runtime.Serialization; namespace OpenAI.Chat diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Content.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Content.cs index 389e49ed..b3d7c80e 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Content.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Content.cs @@ -1,4 +1,8 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using Newtonsoft.Json; +using System; +using UnityEngine; using UnityEngine.Scripting; namespace OpenAI.Chat @@ -6,6 +10,22 @@ namespace OpenAI.Chat [Preserve] public sealed class Content { + [Preserve] + public static implicit operator Content(string content) => new Content(content); + + [Preserve] + public Content(string text) + : this(ContentType.Text, text) + { + } + + [Preserve] + public Content(Texture2D texture) + : this(ContentType.ImageUrl, $"data:image/jpeg;base64,{Convert.ToBase64String(texture.EncodeToPNG())}") + { + } + + [Preserve] public Content(ContentType type, string input) { Type = type; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ContentType.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ContentType.cs index 79a75059..fe008e60 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ContentType.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ContentType.cs @@ -1,3 +1,5 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using System.Runtime.Serialization; namespace OpenAI.Chat diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Conversation.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Conversation.cs index 8ad2f75c..93e72941 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Conversation.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Conversation.cs @@ -28,10 +28,13 @@ public Conversation([JsonProperty("messages")] List messages) /// Appends to the end of . /// /// The message to add to the . + [Preserve] public void AppendMessage(Message message) => messages.Add(message); + [Preserve] public override string ToString() => JsonConvert.SerializeObject(this, OpenAIClient.JsonSerializationOptions); + [Preserve] public static implicit operator string(Conversation conversation) => conversation.ToString(); } } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/FinishDetails.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/FinishDetails.cs index 8f2ec185..0855029a 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/FinishDetails.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/FinishDetails.cs @@ -1,3 +1,5 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using Newtonsoft.Json; using UnityEngine.Scripting; @@ -6,6 +8,9 @@ namespace OpenAI.Chat [Preserve] public sealed class FinishDetails { + [Preserve] + public FinishDetails() { } + [Preserve] [JsonProperty("type")] public string Type { get; private set; } diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ImageUrl.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ImageUrl.cs index f08518bd..f34b98cf 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ImageUrl.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ImageUrl.cs @@ -1,3 +1,5 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using Newtonsoft.Json; using UnityEngine.Scripting; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Message.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Message.cs index 5d43f400..2a8e4776 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Message.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Message.cs @@ -114,26 +114,25 @@ public Role Role private set => role = value; } - [Preserve] [SerializeField] [TextArea(1, 30)] private string content; - private dynamic contentList; + private object contentList; /// /// The contents of the message. /// [Preserve] [JsonProperty("content", DefaultValueHandling = DefaultValueHandling.Populate, NullValueHandling = NullValueHandling.Include, Required = Required.AllowNull)] - public dynamic Content + public object Content { get => contentList ?? content; private set { - if (value is string) + if (value is string s) { - content = value; + content = s; } else { diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ResponseFormat.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ResponseFormat.cs index 93dce2af..d6e452e4 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/ResponseFormat.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/ResponseFormat.cs @@ -1,3 +1,5 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using Newtonsoft.Json; using UnityEngine.Scripting; diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Tool.cs b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Tool.cs index bed4e182..2855de79 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Chat/Tool.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Chat/Tool.cs @@ -1,14 +1,20 @@ +// Licensed under the MIT License. See LICENSE in the project root for license information. + using Newtonsoft.Json; using UnityEngine.Scripting; namespace OpenAI.Chat { + [Preserve] public sealed class Tool { + [Preserve] public Tool() { } + [Preserve] public Tool(Tool other) => CopyFrom(other); + [Preserve] public Tool(Function function) { Function = function; @@ -31,8 +37,10 @@ public Tool(Function function) [JsonProperty("function")] public Function Function { get; private set; } + [Preserve] public static implicit operator Tool(Function function) => new Tool(function); + [Preserve] internal void CopyFrom(Tool other) { if (!string.IsNullOrWhiteSpace(other?.Id)) diff --git a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/StringExtensions.cs b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/StringExtensions.cs index 072244f8..2e68bd45 100644 --- a/OpenAI/Packages/com.openai.unity/Runtime/Extensions/StringExtensions.cs +++ b/OpenAI/Packages/com.openai.unity/Runtime/Extensions/StringExtensions.cs @@ -23,14 +23,14 @@ public static string CreateNewDirectory(this string parentDirectory, string newD throw new ArgumentNullException(nameof(newDirectoryName)); } - var voiceDirectory = Path.Combine(parentDirectory, newDirectoryName); + var directory = Path.Combine(parentDirectory, newDirectoryName); - if (!Directory.Exists(voiceDirectory)) + if (!Directory.Exists(directory)) { - Directory.CreateDirectory(voiceDirectory); + Directory.CreateDirectory(directory); } - return voiceDirectory; + return directory; } } } diff --git a/OpenAI/Packages/com.openai.unity/Samples~/Chat/ChatBehaviour.cs b/OpenAI/Packages/com.openai.unity/Samples~/Chat/ChatBehaviour.cs index 7493d07d..94d9a5d3 100644 --- a/OpenAI/Packages/com.openai.unity/Samples~/Chat/ChatBehaviour.cs +++ b/OpenAI/Packages/com.openai.unity/Samples~/Chat/ChatBehaviour.cs @@ -1,5 +1,6 @@ // Licensed under the MIT License. See LICENSE in the project root for license information. +using OpenAI.Audio; using OpenAI.Chat; using OpenAI.Models; using System; @@ -15,6 +16,9 @@ namespace OpenAI.Samples.Chat { public class ChatBehaviour : MonoBehaviour { + [SerializeField] + private bool enableDebug; + [SerializeField] private Button submitButton; @@ -27,6 +31,9 @@ public class ChatBehaviour : MonoBehaviour [SerializeField] private ScrollRect scrollView; + [SerializeField] + private AudioSource audioSource; + private OpenAIClient openAI; private readonly List chatMessages = new List(); @@ -38,6 +45,7 @@ private void OnValidate() inputField.Validate(); contentArea.Validate(); submitButton.Validate(); + audioSource.Validate(); } private void Awake() @@ -74,22 +82,20 @@ private async void SubmitChat() var userMessageContent = AddNewTextMessageContent(); userMessageContent.text = $"User: {inputField.text}"; inputField.text = string.Empty; - var assistantMessageContent = AddNewTextMessageContent(); assistantMessageContent.text = "Assistant: "; try { - await openAI.ChatEndpoint.StreamCompletionAsync( - new ChatRequest(chatMessages, Model.GPT3_5_Turbo), - response => - { - if (response.FirstChoice?.Delta != null) - { - assistantMessageContent.text += response.ToString(); - scrollView.verticalNormalizedPosition = 0f; - } - }, lifetimeCancellationTokenSource.Token); + var request = new ChatRequest(chatMessages, Model.GPT3_5_Turbo); + openAI.ChatEndpoint.EnableDebug = enableDebug; + var response = await openAI.ChatEndpoint.StreamCompletionAsync(request, resultHandler: deltaResponse => + { + if (deltaResponse?.FirstChoice?.Delta == null) { return; } + assistantMessageContent.text += deltaResponse.FirstChoice.Delta.ToString(); + scrollView.verticalNormalizedPosition = 0f; + }, lifetimeCancellationTokenSource.Token); + GenerateSpeech(response); } catch (Exception e) { @@ -97,7 +103,7 @@ await openAI.ChatEndpoint.StreamCompletionAsync( } finally { - if (lifetimeCancellationTokenSource != null) + if (lifetimeCancellationTokenSource is { IsCancellationRequested: false }) { inputField.interactable = true; EventSystem.current.SetSelectedGameObject(inputField.gameObject); @@ -108,6 +114,22 @@ await openAI.ChatEndpoint.StreamCompletionAsync( } } + private async void GenerateSpeech(ChatResponse response) + { + try + { + var request = new SpeechRequest(response.FirstChoice.ToString(), Model.TTS_1); + openAI.AudioEndpoint.EnableDebug = enableDebug; + var (clipPath, clip) = await openAI.AudioEndpoint.CreateSpeechAsync(request, lifetimeCancellationTokenSource.Token); + audioSource.PlayOneShot(clip); + Debug.Log(clipPath); + } + catch (Exception e) + { + Debug.LogError(e); + } + } + private TextMeshProUGUI AddNewTextMessageContent() { var textObject = new GameObject($"Message_{contentArea.childCount + 1}"); diff --git a/OpenAI/Packages/com.openai.unity/Samples~/Chat/OpenAIChatSample.unity b/OpenAI/Packages/com.openai.unity/Samples~/Chat/OpenAIChatSample.unity index 358d0986..ef378a72 100644 --- a/OpenAI/Packages/com.openai.unity/Samples~/Chat/OpenAIChatSample.unity +++ b/OpenAI/Packages/com.openai.unity/Samples~/Chat/OpenAIChatSample.unity @@ -104,7 +104,7 @@ NavMeshSettings: serializedVersion: 2 m_ObjectHideFlags: 0 m_BuildSettings: - serializedVersion: 2 + serializedVersion: 3 agentTypeID: 0 agentRadius: 0.5 agentHeight: 2 @@ -117,7 +117,7 @@ NavMeshSettings: cellSize: 0.16666667 manualTileSize: 0 tileSize: 256 - accuratePlacement: 0 + buildHeightMesh: 0 maxJobWorkers: 0 preserveTilesOutsideBounds: 0 debug: @@ -156,7 +156,6 @@ RectTransform: m_Children: - {fileID: 250955499} m_Father: {fileID: 1974642465} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -245,7 +244,6 @@ RectTransform: m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 235166} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 1} m_AnchorMax: {x: 1, y: 1} @@ -323,7 +321,6 @@ RectTransform: m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 942593597} - m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -458,7 +455,6 @@ RectTransform: m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 1466169039} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -534,7 +530,6 @@ RectTransform: m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 1094024332} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -668,7 +663,6 @@ RectTransform: m_Children: - {fileID: 800336257} m_Father: {fileID: 1819767326} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -707,7 +701,6 @@ RectTransform: - {fileID: 1377121431} - {fileID: 1094024332} m_Father: {fileID: 996239086} - m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 0} @@ -773,7 +766,6 @@ RectTransform: m_Children: - {fileID: 1466169039} m_Father: {fileID: 1974642465} - m_RootOrder: 2 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 1, y: 0} m_AnchorMax: {x: 1, y: 0} @@ -899,7 +891,6 @@ RectTransform: m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 942593597} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -1054,7 +1045,6 @@ RectTransform: m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 619328969} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -1131,7 +1121,6 @@ RectTransform: - {fileID: 768762704} - {fileID: 334289164} m_Father: {fileID: 1377121431} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -1185,7 +1174,6 @@ RectTransform: - {fileID: 1974642465} - {fileID: 658807647} m_Father: {fileID: 1711080860} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -1264,7 +1252,6 @@ RectTransform: m_Children: - {fileID: 530667793} m_Father: {fileID: 658807647} - m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -1433,13 +1420,13 @@ Transform: m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 1246159954} + serializedVersion: 2 m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} m_LocalPosition: {x: 0, y: 0, z: 0} m_LocalScale: {x: 1, y: 1, z: 1} m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 0} - m_RootOrder: 3 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &1287381581 GameObject: @@ -1527,13 +1514,13 @@ Transform: m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 1287381581} + serializedVersion: 2 m_LocalRotation: {x: 0.40821788, y: -0.23456968, z: 0.10938163, w: 0.8754261} m_LocalPosition: {x: 0, y: 3, z: 0} m_LocalScale: {x: 1, y: 1, z: 1} m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 0} - m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 50, y: -30, z: 0} --- !u!1 &1358986983 GameObject: @@ -1575,9 +1562,17 @@ Camera: m_projectionMatrixMode: 1 m_GateFitMode: 2 m_FOVAxisMode: 0 + m_Iso: 200 + m_ShutterSpeed: 0.005 + m_Aperture: 16 + m_FocusDistance: 10 + m_FocalLength: 50 + m_BladeCount: 5 + m_Curvature: {x: 2, y: 11} + m_BarrelClipping: 0.25 + m_Anamorphism: 0 m_SensorSize: {x: 36, y: 24} m_LensShift: {x: 0, y: 0} - m_FocalLength: 50 m_NormalizedViewPortRect: serializedVersion: 2 x: 0 @@ -1611,13 +1606,13 @@ Transform: m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 1358986983} + serializedVersion: 2 m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} m_LocalPosition: {x: 0, y: 1, z: -10} m_LocalScale: {x: 1, y: 1, z: 1} m_ConstrainProportionsScale: 1 m_Children: [] m_Father: {fileID: 0} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &1377121430 GameObject: @@ -1653,7 +1648,6 @@ RectTransform: m_Children: - {fileID: 942593597} m_Father: {fileID: 658807647} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -1846,7 +1840,6 @@ RectTransform: m_Children: - {fileID: 422726883} m_Father: {fileID: 740935985} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -1866,6 +1859,7 @@ GameObject: - component: {fileID: 1711080858} - component: {fileID: 1711080857} - component: {fileID: 1711080861} + - component: {fileID: 1711080862} m_Layer: 5 m_Name: Canvas m_TagString: Untagged @@ -1930,7 +1924,9 @@ Canvas: m_OverrideSorting: 0 m_OverridePixelPerfect: 0 m_SortingBucketNormalizedSize: 0 + m_VertexColorAlwaysGammaSpace: 0 m_AdditionalShaderChannelsFlag: 25 + m_UpdateRectTransformForStandalone: 0 m_SortingLayerID: 0 m_SortingOrder: 0 m_TargetDisplay: 0 @@ -1948,7 +1944,6 @@ RectTransform: m_Children: - {fileID: 996239086} m_Father: {fileID: 0} - m_RootOrder: 2 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -1967,10 +1962,108 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: a891710bf1466924297c3b3b6f1b6e51, type: 3} m_Name: m_EditorClassIdentifier: + enableDebug: 0 submitButton: {fileID: 1094024334} inputField: {fileID: 1377121433} contentArea: {fileID: 250955499} scrollView: {fileID: 1974642466} + audioSource: {fileID: 1711080862} +--- !u!82 &1711080862 +AudioSource: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1711080856} + m_Enabled: 1 + serializedVersion: 4 + OutputAudioMixerGroup: {fileID: 0} + m_audioClip: {fileID: 0} + m_PlayOnAwake: 0 + m_Volume: 1 + m_Pitch: 1 + Loop: 0 + Mute: 0 + Spatialize: 0 + SpatializePostEffects: 0 + Priority: 128 + DopplerLevel: 1 + MinDistance: 1 + MaxDistance: 500 + Pan2D: 0 + rolloffMode: 0 + BypassEffects: 0 + BypassListenerEffects: 0 + BypassReverbZones: 0 + rolloffCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 1 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + - serializedVersion: 3 + time: 1 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + panLevelCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + spreadCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 0 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 + reverbZoneMixCustomCurve: + serializedVersion: 2 + m_Curve: + - serializedVersion: 3 + time: 0 + value: 1 + inSlope: 0 + outSlope: 0 + tangentMode: 0 + weightedMode: 0 + inWeight: 0.33333334 + outWeight: 0.33333334 + m_PreInfinity: 2 + m_PostInfinity: 2 + m_RotationOrder: 4 --- !u!1 &1819767325 GameObject: m_ObjectHideFlags: 0 @@ -2004,7 +2097,6 @@ RectTransform: m_Children: - {fileID: 619328969} m_Father: {fileID: 1974642465} - m_RootOrder: 1 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} @@ -2131,7 +2223,6 @@ RectTransform: - {fileID: 1819767326} - {fileID: 740935985} m_Father: {fileID: 996239086} - m_RootOrder: 0 m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 1, y: 1} @@ -2168,3 +2259,11 @@ MonoBehaviour: m_OnValueChanged: m_PersistentCalls: m_Calls: [] +--- !u!1660057539 &9223372036854775807 +SceneRoots: + m_ObjectHideFlags: 0 + m_Roots: + - {fileID: 1358986986} + - {fileID: 1287381583} + - {fileID: 1711080860} + - {fileID: 1246159957} diff --git a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_03_Chat.cs b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_03_Chat.cs index b12d1179..c890fca7 100644 --- a/OpenAI/Packages/com.openai.unity/Tests/TestFixture_03_Chat.cs +++ b/OpenAI/Packages/com.openai.unity/Tests/TestFixture_03_Chat.cs @@ -9,7 +9,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using UnityEditor; using UnityEngine; +using Tool = OpenAI.Chat.Tool; namespace OpenAI.Tests { @@ -82,7 +84,7 @@ public async Task Test_02_GetChatStreamingCompletion() for (var i = 0; i < choiceCount; i++) { var choice = response.Choices[i]; - Assert.IsFalse(string.IsNullOrEmpty(choice?.Message?.Content)); + Assert.IsFalse(string.IsNullOrEmpty(choice?.Message?.ToString())); Debug.Log($"[{choice.Index}] {choice.Message.Role}: {choice.Message.Content} | Finish Reason: {choice.FinishReason}"); Assert.IsTrue(choice.Message.Role == Role.Assistant); var deltaContent = cumulativeDelta[i]; @@ -154,7 +156,7 @@ public async Task Test_04_GetChatFunctionCompletion() Assert.IsTrue(result.Choices.Count == 1); messages.Add(result.FirstChoice.Message); - if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content)) + if (!string.IsNullOrEmpty(result.FirstChoice.Message.ToString())) { Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); @@ -236,7 +238,7 @@ public async Task Test_05_GetChatFunctionCompletion_Streaming() Debug.Log($"{choice.Delta.Content}"); } - foreach (var choice in partialResponse.Choices.Where(choice => !string.IsNullOrEmpty(choice.Message?.Content))) + foreach (var choice in partialResponse.Choices.Where(choice => !string.IsNullOrEmpty(choice.Message?.Content?.ToString()))) { Debug.Log($"{choice.Message.Role}: {choice.Message.Content} | Finish Reason: {choice.FinishReason}"); } @@ -261,7 +263,7 @@ public async Task Test_05_GetChatFunctionCompletion_Streaming() Debug.Log($"[{choice.Index}] {choice.Delta.Content}"); } - foreach (var choice in partialResponse.Choices.Where(choice => !string.IsNullOrEmpty(choice.Message?.Content))) + foreach (var choice in partialResponse.Choices.Where(choice => !string.IsNullOrEmpty(choice.Message?.Content?.ToString()))) { Debug.Log($"[{choice.Index}] {choice.Message.Role}: {choice.Message.Content} | Finish Reason: {choice.FinishReason}"); } @@ -271,7 +273,7 @@ public async Task Test_05_GetChatFunctionCompletion_Streaming() Assert.IsTrue(result.Choices.Count == 1); messages.Add(result.FirstChoice.Message); - if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content)) + if (!string.IsNullOrEmpty(result.FirstChoice.Message?.ToString())) { Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); @@ -290,7 +292,7 @@ public async Task Test_05_GetChatFunctionCompletion_Streaming() Debug.Log($"{choice.Delta.Content}"); } - foreach (var choice in partialResponse.Choices.Where(choice => !string.IsNullOrEmpty(choice.Message?.Content))) + foreach (var choice in partialResponse.Choices.Where(choice => !string.IsNullOrEmpty(choice.Message?.ToString()))) { Debug.Log($"{choice.Message.Role}: {choice.Message.Content} | Finish Reason: {choice.FinishReason}"); } @@ -452,7 +454,7 @@ public async Task Test_07_GetChatToolCompletion() Assert.IsTrue(result.Choices.Count == 1); messages.Add(result.FirstChoice.Message); - if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content)) + if (!string.IsNullOrEmpty(result.FirstChoice.Message.ToString())) { Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); @@ -549,7 +551,7 @@ public async Task Test_08_GetChatToolCompletion_Streaming() Assert.IsTrue(result.Choices.Count == 1); messages.Add(result.FirstChoice.Message); - if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content)) + if (!string.IsNullOrEmpty(result.FirstChoice.Message.ToString())) { Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); @@ -705,5 +707,30 @@ public async Task Test_11_GetChatVisionStreaming() Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}"); result.GetUsage(); } + + [Test] + public async Task Test_12_GetChatVision_Texture() + { + var api = new OpenAIClient(OpenAIAuthentication.Default.LoadFromEnvironment()); + Assert.IsNotNull(api.ChatEndpoint); + var imageAssetPath = AssetDatabase.GUIDToAssetPath("230fd778637d3d84d81355c8c13b1999"); + var image = AssetDatabase.LoadAssetAtPath(imageAssetPath); + var messages = new List + { + new Message(Role.System, "You are a helpful assistant."), + new Message(Role.User, new List + { + new Content(ContentType.Text, "What's in this image?"), + new Content(image) + }) + }; + var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview"); + var result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); + Assert.IsNotNull(result); + Assert.IsNotNull(result.Choices); + Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}"); + result.GetUsage(); + } + } } diff --git a/OpenAI/Packages/com.openai.unity/package.json b/OpenAI/Packages/com.openai.unity/package.json index db298014..b83a1322 100644 --- a/OpenAI/Packages/com.openai.unity/package.json +++ b/OpenAI/Packages/com.openai.unity/package.json @@ -3,7 +3,7 @@ "displayName": "OpenAI", "description": "A OpenAI package for the Unity Game Engine to use GPT-4, GPT-3.5, GPT-3 and Dall-E though their RESTful API (currently in beta).\n\nIndependently developed, this is not an official library and I am not affiliated with OpenAI.\n\nAn OpenAI API account is required.", "keywords": [], - "version": "5.2.0", + "version": "5.2.1", "unity": "2021.3", "documentationUrl": "https://github.com/RageAgainstThePixel/com.openai.unity#documentation", "changelogUrl": "https://github.com/RageAgainstThePixel/com.openai.unity/releases", diff --git a/OpenAI/ProjectSettings/ProjectSettings.asset b/OpenAI/ProjectSettings/ProjectSettings.asset index 6464a0ad..ff149e76 100644 --- a/OpenAI/ProjectSettings/ProjectSettings.asset +++ b/OpenAI/ProjectSettings/ProjectSettings.asset @@ -59,9 +59,9 @@ PlayerSettings: androidShowActivityIndicatorOnLoading: -1 iosUseCustomAppBackgroundBehavior: 0 allowedAutorotateToPortrait: 1 - allowedAutorotateToPortraitUpsideDown: 1 - allowedAutorotateToLandscapeRight: 1 - allowedAutorotateToLandscapeLeft: 1 + allowedAutorotateToPortraitUpsideDown: 0 + allowedAutorotateToLandscapeRight: 0 + allowedAutorotateToLandscapeLeft: 0 useOSAutorotation: 1 use32BitDisplayBuffer: 1 preserveFramebufferAlpha: 0 @@ -136,7 +136,7 @@ PlayerSettings: vulkanEnableLateAcquireNextImage: 0 vulkanEnableCommandBufferRecycling: 1 loadStoreDebugModeEnabled: 0 - bundleVersion: 5.0.0 + bundleVersion: 5.2.1 preloadedAssets: [] metroInputSource: 0 wsaTransparentSwapchain: 0 @@ -782,7 +782,7 @@ PlayerSettings: m_RenderingPath: 1 m_MobileRenderingPath: 1 metroPackageName: com.openai.unity - metroPackageVersion: 5.0.0.0 + metroPackageVersion: 5.2.1.0 metroCertificatePath: metroCertificatePassword: metroCertificateSubject: @@ -894,7 +894,7 @@ PlayerSettings: luminIsChannelApp: 0 luminVersion: m_VersionCode: 1 - m_VersionName: 5.0.0 + m_VersionName: 5.2.1 hmiPlayerDataPath: hmiForceSRGBBlit: 1 embeddedLinuxEnableGamepadInput: 1 diff --git a/README.md b/README.md index c7704f8b..0ead356e 100644 --- a/README.md +++ b/README.md @@ -62,31 +62,32 @@ The recommended installation method is though the unity package manager and [Ope - [Chat](#chat) - [Chat Completions](#chat-completions) - [Streaming](#chat-streaming) - - [Functions](#chat-functions) + - [Tools](#chat-tools) :new: + - [Vision](#chat-vision) :new: - [Edits](#edits) - [Create Edit](#create-edit) - [Embeddings](#embeddings) - [Create Embedding](#create-embeddings) -- [Audio](#audio) :construction: - - [Create Speech](#create-speech) :new: +- [Audio](#audio) + - [Create Speech](#create-speech) - [Create Transcription](#create-transcription) - [Create Translation](#create-translation) -- [Images](#images) :construction: - - [Create Image](#create-image) :new: - - [Edit Image](#edit-image) :new: - - [Create Image Variation](#create-image-variation) :new: +- [Images](#images) + - [Create Image](#create-image) + - [Edit Image](#edit-image) + - [Create Image Variation](#create-image-variation) - [Files](#files) - [List Files](#list-files) - [Upload File](#upload-file) - [Delete File](#delete-file) - [Retrieve File Info](#retrieve-file-info) - [Download File Content](#download-file-content) -- [Fine Tuning](#fine-tuning) :construction: - - [Create Fine Tune Job](#create-fine-tune-job) :new: - - [List Fine Tune Jobs](#list-fine-tune-jobs) :new: - - [Retrieve Fine Tune Job Info](#retrieve-fine-tune-job-info) :new: - - [Cancel Fine Tune Job](#cancel-fine-tune-job) :new: - - [List Fine Tune Job Events](#list-fine-tune-job-events) :new: +- [Fine Tuning](#fine-tuning) + - [Create Fine Tune Job](#create-fine-tune-job) + - [List Fine Tune Jobs](#list-fine-tune-jobs) + - [Retrieve Fine Tune Job Info](#retrieve-fine-tune-job-info) + - [Cancel Fine Tune Job](#cancel-fine-tune-job) + - [List Fine Tune Job Events](#list-fine-tune-job-events) - [Moderations](#moderations) - [Create Moderation](#create-moderation) @@ -99,6 +100,8 @@ There are 4 ways to provide your API keys, in order of precedence: 3. [Load key from configuration file](#load-key-from-configuration-file) 4. [Use System Environment Variables](#use-system-environment-variables) +You use the `OpenAIAuthentication` when you initialize the API as shown: + #### Pass keys directly with constructor :warning: We recommended using the environment variables to load the API key instead of having it hard coded in your source. It is not recommended use this method in production, but only for accepting user credentials, local testing and quick start scenarios. @@ -408,7 +411,7 @@ await api.ChatEndpoint.StreamCompletionAsync(chatRequest, result => }); ``` -##### [Chat Functions](https://platform.openai.com/docs/api-reference/chat/create#chat/create-functions) +##### [Chat Tools](https://platform.openai.com/docs/guides/function-calling) > Only available with the latest 0613 model series! @@ -425,8 +428,8 @@ foreach (var message in messages) Debug.Log($"{message.Role}: {message.Content}"); } -// Define the functions that the assistant is able to use: -var functions = new List +// Define the tools that the assistant is able to use: +var tools = new List { new Function( nameof(WeatherService.GetCurrentWeather), @@ -451,45 +454,88 @@ var functions = new List }) }; -var chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo"); +var chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto"); var result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); messages.Add(result.FirstChoice.Message); + Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); + var locationMessage = new Message(Role.User, "I'm in Glasgow, Scotland"); messages.Add(locationMessage); Debug.Log($"{locationMessage.Role}: {locationMessage.Content}"); -chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo"); +chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto"); result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); + messages.Add(result.FirstChoice.Message); if (!string.IsNullOrEmpty(result.FirstChoice.Message.Content)) { - // It's possible that the assistant will also ask you which units you want the temperature in. Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishReason}"); var unitMessage = new Message(Role.User, "celsius"); messages.Add(unitMessage); Debug.Log($"{unitMessage.Role}: {unitMessage.Content}"); - chatRequest = new ChatRequest(messages, functions: functions, functionCall: "auto", model: "gpt-3.5-turbo"); + chatRequest = new ChatRequest(messages, tools: tools, toolChoice: "auto"); result = await api.ChatEndpoint.GetCompletionAsync(chatRequest); } -Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Function.Name} | Finish Reason: {result.FirstChoice.FinishReason}"); -Debug.Log($"{result.FirstChoice.Message.Function.Arguments}"); -var functionArgs = JsonConvert.DeserializeObject(result.FirstChoice.Message.Function.Arguments.ToString()); +var usedTool = result.FirstChoice.Message.ToolCalls[0]; +Debug.Log($"{result.FirstChoice.Message.Role}: {usedTool.Function.Name} | Finish Reason: {result.FirstChoice.FinishReason}"); +Debug.Log($"{usedTool.Function.Arguments}"); +var functionArgs = JsonSerializer.Deserialize(usedTool.Function.Arguments.ToString()); var functionResult = WeatherService.GetCurrentWeather(functionArgs); -messages.Add(new Message(Role.Function, functionResult, nameof(WeatherService.GetCurrentWeather))); -Debug.Log($"{Role.Function}: {functionResult}"); +messages.Add(new Message(usedTool, functionResult)); +Debug.Log($"{Role.Tool}: {functionResult}"); // System: You are a helpful weather assistant. // User: What's the weather like today? // Assistant: Sure, may I know your current location? | Finish Reason: stop // User: I'm in Glasgow, Scotland -// Assistant: GetCurrentWeather | Finish Reason: function_call +// Assistant: GetCurrentWeather | Finish Reason: tool_calls // { // "location": "Glasgow, Scotland", // "unit": "celsius" // } -// Function: The current weather in Glasgow, Scotland is 20 celsius +// Tool: The current weather in Glasgow, Scotland is 20 celsius +``` + +##### [Chat Vision](https://platform.openai.com/docs/guides/vision) + +:construction: This feature is in beta! + +> Currently, GPT-4 with vision does not support the message.name parameter, functions/tools, nor the response_format parameter. + +```csharp +var api = new OpenAIClient(); +var messages = new List +{ + new Message(Role.System, "You are a helpful assistant."), + new Message(Role.User, new List + { + new Content(ContentType.Text, "What's in this image?"), + new Content(ContentType.ImageUrl, "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg") + }) +}; +var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview"); +var result = await apiChatEndpoint.GetCompletionAsync(chatRequest); +Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}"); +``` + +You can even pass in a `Texture2D`! + +```csharp +var api = new OpenAIClient(); +var messages = new List +{ + new Message(Role.System, "You are a helpful assistant."), + new Message(Role.User, new List + { + new Content(ContentType.Text, "What's in this image?"), + new Content(texture) + }) +}; +var chatRequest = new ChatRequest(messages, model: "gpt-4-vision-preview"); +var result = await apiChatEndpoint.GetCompletionAsync(chatRequest); +Debug.Log($"{result.FirstChoice.Message.Role}: {result.FirstChoice.Message.Content} | Finish Reason: {result.FirstChoice.FinishDetails}"); ``` ### [Edits](https://platform.openai.com/docs/api-reference/edits)