Skip to content

Commit

Permalink
[IngestionClient] Use Isolated Worker model in .Net 8.0 (#2482)
Browse files Browse the repository at this point in the history
  • Loading branch information
ross-p-smith authored Jul 10, 2024
1 parent b329c0a commit 4c4f8ea
Show file tree
Hide file tree
Showing 28 changed files with 596 additions and 143 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"vscode": {
"extensions": [
"ms-dotnettools.csdevkit",
"ms-vscode.azure-account",
"ms-azuretools.vscode-azurefunctions",
"editorconfig.editorconfig"
],
Expand Down
1 change: 0 additions & 1 deletion samples/ingestion/ingestion-client/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
.vscode
local.settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@
<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" Version="$(AzureStorageBlobsPackageVersion)" />
<PackageReference Include="JsonSubTypes" Version="$(JsonSubTypesPackageVersion)" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="$(MicrosoftAzureWebJobsExtensionsServiceBusPackageVersion)" />
<PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="$(MicrosoftApplicationInsightsWorkerServicePackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="$(MicrosoftAzureFunctionsWorkerPackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="$(MicrosoftAzureFunctionsWorkerApplicationInsightsPackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="$(MicrosoftAzureFunctionsExtensionsPackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.ServiceBus" Version="$(MicrosoftAzureFunctionsWorkerExtensionsServiceBusPackageVersion)" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="$(MicrosoftEntityFrameworkCorePackageVersion)" />
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="$(MicrosoftEntityFrameworkCoreSqlServerPackageVersion)" />
<PackageReference Include="Polly" Version="$(PollyPackageVersion)" />
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// <copyright file="AppInsightsServiceCollectionExtensions.cs" company="Microsoft Corporation">
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
// </copyright>

namespace Connector
{
using System.Linq;

using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

/// <summary>
/// Extension methods for configuring Application Insights.
/// </summary>
public static class AppInsightsServiceCollectionExtensions
{
/// <summary>
/// Configures the logger for the Ingestion Client and removes the default logging filter added by the Application Insights SDK.
/// </summary>
/// <param name="services">The Service Collection</param>
/// <returns>The Service Collection</returns>
public static IServiceCollection ConfigureIngestionClientLogging(this IServiceCollection services)
{
services.AddApplicationInsightsTelemetryWorkerService();
services.ConfigureFunctionsApplicationInsights();

services.Configure<LoggerFilterOptions>(options =>
{
// The Application Insights SDK adds a default logging filter that instructs ILogger to capture only Warning and more severe logs. Application Insights requires an explicit override.
// Log levels can also be configured using appsettings.json. For more information, see https://learn.microsoft.com/en-us/azure/azure-monitor/app/worker-service#ilogger-logs
LoggerFilterRule toRemove = options.Rules.FirstOrDefault(rule => rule.ProviderName
== "Microsoft.Extensions.Logging.ApplicationInsights.ApplicationInsightsLoggerProvider");
if (toRemove is not null)
{
options.Rules.Remove(toRemove);
}
});

return services;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Attach to Fetch Transcription from Queue",
"type": "coreclr",
"request": "attach",
"processId": "${command:azureFunctions.pickProcess}"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"azureFunctions.deploySubpath": "bin/Release/net8.0/publish",
"azureFunctions.projectLanguage": "C#",
"azureFunctions.projectRuntime": "~4",
"azureFunctions.preDeployTask": "publish (functions)"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "clean (functions)",
"command": "dotnet",
"args": [
"clean",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"problemMatcher": "$msCompile"
},
{
"label": "build (functions)",
"command": "dotnet",
"args": [
"build",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"dependsOn": "clean (functions)",
"group": {
"kind": "build",
"isDefault": true
},
"problemMatcher": "$msCompile"
},
{
"label": "clean release (functions)",
"command": "dotnet",
"args": [
"clean",
"--configuration",
"Release",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"problemMatcher": "$msCompile"
},
{
"label": "publish (functions)",
"command": "dotnet",
"args": [
"publish",
"--configuration",
"Release",
"/property:GenerateFullPaths=true",
"/consoleloggerparameters:NoSummary"
],
"type": "process",
"dependsOn": "clean release (functions)",
"problemMatcher": "$msCompile"
},
{
"type": "func",
"dependsOn": "build (functions)",
"options": {
"cwd": "${workspaceFolder}/bin/Debug/net8.0"
},
"command": "host start --port 7072",
"isBackground": true,
"problemMatcher": "$func-dotnet-watch"
}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,39 +9,51 @@ namespace FetchTranscription
using System.Threading.Tasks;
using Connector;

using Microsoft.Azure.WebJobs;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.Logging;

/// <summary>
/// Fetch Transcription class.
/// </summary>
public class FetchTranscription
{
private readonly IServiceProvider serviceProvider;

public FetchTranscription(IServiceProvider serviceProvider)
private readonly ILogger<FetchTranscription> logger;

/// <summary>
/// Initializes a new instance of the <see cref="FetchTranscription"/> class.
/// </summary>
/// <param name="serviceProvider">The service provider.</param>
/// <param name="logger">The FetchTranscription logger.</param>
public FetchTranscription(IServiceProvider serviceProvider, ILogger<FetchTranscription> logger)
{
this.serviceProvider = serviceProvider;
this.logger = logger;
}

[FunctionName("FetchTranscription")]
public async Task Run([ServiceBusTrigger("fetch_transcription_queue", Connection = "AzureServiceBus")]string message, ILogger log)
/// <summary>
/// Triggered by a Service Bus message to fetch the transcription.
/// </summary>
/// <param name="message">The message on the queue</param>
[Function("FetchTranscription")]
public async Task Run([ServiceBusTrigger("fetch_transcription_queue", Connection = "AzureServiceBus")]string message)
{
if (log == null)
{
throw new ArgumentNullException(nameof(log));
}
ArgumentNullException.ThrowIfNull(this.logger, nameof(this.logger));

log.LogInformation($"C# Service bus triggered function executed at: {DateTime.Now}");
this.logger.LogInformation($"C# Isolated Service bus triggered function executed at: {DateTime.Now}");

if (string.IsNullOrEmpty(message))
{
log.LogInformation($"Found invalid service bus message: {message}. Stopping execution.");
this.logger.LogInformation($"Found invalid service bus message: {message}. Stopping execution.");
return;
}

var serviceBusMessage = TranscriptionStartedMessage.DeserializeMessage(message);

var transcriptionProcessor = new TranscriptionProcessor(this.serviceProvider);

await transcriptionProcessor.ProcessTranscriptionJobAsync(serviceBusMessage, this.serviceProvider, log).ConfigureAwait(false);
await transcriptionProcessor.ProcessTranscriptionJobAsync(serviceBusMessage, this.serviceProvider, this.logger).ConfigureAwait(false);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<AzureFunctionsVersion>v4</AzureFunctionsVersion>
<OutputType>Exe</OutputType>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Azure.AI.Language.Conversations" Version="$(AzureAILanguageConversationsPackageVersion)" />
<PackageReference Include="Azure.AI.TextAnalytics" Version="$(AzureAITextAnalyticsPackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="$(MicrosoftAzureFunctionsWorkerPackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="$(MicrosoftAzureFunctionsExtensionsPackageVersion)" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="$(MicrosoftAzureWebJobsExtensionsServiceBusPackageVersion)" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="$(MicrosoftNETSdkFunctionsPackageVersion)" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="$(MicrosoftAzureFunctionsWorkerSdkPackageVersion)" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Connector\Connector.csproj" />
Expand All @@ -21,6 +22,7 @@
<ItemGroup>
<None Update="local.settings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
</None>
</ItemGroup>
</Project>
39 changes: 39 additions & 0 deletions samples/ingestion/ingestion-client/FetchTranscription/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// <copyright file="Program.cs" company="Microsoft Corporation">
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
// </copyright>

namespace FetchTranscription
{
using System.Threading.Tasks;

using Connector;
using Connector.Database;

using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;

public static class Program
{
public static async Task Main(string[] args)
{
var host = new HostBuilder()
.ConfigureFunctionsWorkerDefaults()
.ConfigureServices(s =>
{
// This is a unified way to configure logging filter for all functions.
s.ConfigureIngestionClientLogging();

if (FetchTranscriptionEnvironmentVariables.UseSqlDatabase)
{
s.AddDbContext<IngestionClientDbContext>(
options => SqlServerDbContextOptionsExtensions.UseSqlServer(options, FetchTranscriptionEnvironmentVariables.DatabaseConnectionString));
}
})
.Build();

await host.RunAsync();
}
}
}
33 changes: 0 additions & 33 deletions samples/ingestion/ingestion-client/FetchTranscription/Startup.cs

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@
},
"variables":
{
"Version": "v2.0.17",
"Version": "v2.1.0",
"AudioInputContainer": "audio-input",
"AudioProcessedContainer": "audio-processed",
"ErrorFilesOutputContainer": "audio-failed",
Expand Down Expand Up @@ -938,7 +938,7 @@
"FetchTranscriptionServiceBusConnectionString": "[listKeys(resourceId('Microsoft.ServiceBus/namespaces/queues/authorizationRules', variables('ServiceBusName'), 'fetch_transcription_queue','FetchTranscription'), '2017-04-01').primaryConnectionString]",
"FilesPerTranscriptionJob": "[variables('FilesPerTranscriptionJob')]",
"FUNCTIONS_EXTENSION_VERSION": "~4",
"FUNCTIONS_WORKER_RUNTIME": "dotnet",
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"FUNCTIONS_INPROC_NET8_ENABLED": 1,
"AzureSpeechServicesEndpointUri": "[variables('AzureSpeechServicesEndpointUri')]",
"InitialPollingDelayInMinutes": "[variables('InitialPollingDelayInMinutes')]",
Expand Down Expand Up @@ -1010,7 +1010,7 @@
"ErrorReportOutputContainer": "[variables('ErrorReportOutputContainer')]",
"FetchTranscriptionServiceBusConnectionString": "[listKeys(resourceId('Microsoft.ServiceBus/namespaces/queues/authorizationRules', variables('ServiceBusName'), 'fetch_transcription_queue','FetchTranscription'), '2017-04-01').primaryConnectionString]",
"FUNCTIONS_EXTENSION_VERSION": "~4",
"FUNCTIONS_WORKER_RUNTIME": "dotnet",
"FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
"FUNCTIONS_INPROC_NET8_ENABLED": 1,
"HtmlResultOutputContainer": "[variables('HtmlResultOutputContainer')]",
"InitialPollingDelayInMinutes": "[variables('InitialPollingDelayInMinutes')]",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Attach to Start Transcription by Service Bus",
"type": "coreclr",
"request": "attach",
"processId": "${command:azureFunctions.pickProcess}"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"azureFunctions.deploySubpath": "bin/Release/net8.0/publish",
"azureFunctions.projectLanguage": "C#",
"azureFunctions.projectRuntime": "~4",
"azureFunctions.preDeployTask": "publish (functions)"
}
Loading

0 comments on commit 4c4f8ea

Please sign in to comment.