Merge pull request #48 from edmacdonald/s3-client-singleton

move s3client creation outside imageflow server code

examples/Imageflow.Server.Example/Imageflow.Server.Example.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.Extensions.NETCore.Setup" Version="3.3.101" />
     <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="5.0.0" />
   </ItemGroup>
 

examples/Imageflow.Server.Example/Startup.cs

@@ -13,6 +13,7 @@
 using System.IO;
 using Amazon.S3;
 using Imageflow.Server.HybridCache;
+using Amazon.Runtime;
 
 namespace Imageflow.Server.Example
 {
@@ -30,6 +31,8 @@ public Startup(IConfiguration configuration, IWebHostEnvironment env)
         // This method gets called by the runtime. Use this method to add services to the container.
         public void ConfigureServices(IServiceCollection services)
         {
+            services.AddAWSService<IAmazonS3>();
+
             services.AddControllersWithViews();
 
             // See the README in src/Imageflow.Server.Storage.RemoteReader/ for more advanced configuration
@@ -40,14 +43,19 @@ public void ConfigureServices(IServiceCollection services)
                     SigningKey = "ChangeMe"
                 }
                 .AddPrefix("/remote/"));
-
+
+
+            var s3client = new AmazonS3Client(new AnonymousAWSCredentials(), RegionEndpoint.USEast1);
+            var s3client1 = Configuration.GetAWSOptions("AWS1").CreateServiceClient<IAmazonS3>();
+            var s3client2 = Configuration.GetAWSOptions("AWS2").CreateServiceClient<IAmazonS3>();
+
             // Make S3 containers available at /ri/ and /imageflow-resources/
             // If you use credentials, do not check them into your repository
             // You can call AddImageflowS3Service multiple times for each unique access key
-            services.AddImageflowS3Service(new S3ServiceOptions( null,null)
-                .MapPrefix("/ri/", RegionEndpoint.USEast1, "resizer-images")
-                .MapPrefix("/imageflow-resources/", RegionEndpoint.USWest2, "imageflow-resources")
-                .MapPrefix("/custom-s3client/", () => new AmazonS3Client(), "custom-client", "", false, false)
+            services.AddImageflowS3Service(new S3ServiceOptions()
+                .MapPrefix("/ri/", s3client, "resizer-images", "", false, false)
+                .MapPrefix("/imageflow-resources/", s3client1, "imageflow-resources", "", false, false)
+                .MapPrefix("/default-s3client/", "custom-client")
             );
 
             // Make Azure container available at /azure

src/Imageflow.Server.Storage.S3/PrefixMapping.cs

@@ -3,13 +3,17 @@
 
 namespace Imageflow.Server.Storage.S3
 {
-    internal struct PrefixMapping
+    internal struct PrefixMapping : IDisposable
     {
         internal string Prefix;
-        internal Func<IAmazonS3> ClientFactory;
+        internal IAmazonS3 S3Client;
         internal string Bucket;
         internal string BlobPrefix;
         internal bool IgnorePrefixCase;
-        internal bool LowercaseBlobPath; 
+        internal bool LowercaseBlobPath;
+        public void Dispose()
+        {
+            try { S3Client?.Dispose(); } catch { }
+        }
     }
 }

src/Imageflow.Server.Storage.S3/README.md

@@ -0,0 +1,52 @@
+
+## Overview
+This release moves construction of the S3 client outside of `ImageFlowServer` leveraging the configuration helpers provided by `AWSSDK.Extensions.NETCore.Setup`.  In most cases, this is all that should be needed to configure the S3Service.
+
+```c#
+services.AddAWSService<IAmazonS3>();
+services.AddImageflowS3Service(new S3ServiceOptions()
+  .MapPrefix("/images/", "image-bucket")
+);
+```
+
+## Breaking Change - S3ServiceOptions Constructors
+
+The `S3ServiceOptions` parameterless constructor is now the only constructor, and it no longer assumes anonymous credentials.  If you previously used the default constructor you *might* need to explicitly use anonymous credentials.  If you used either of the other two constructors that accepted credentials, you should now specify them when configuring the S3 client instead.
+```c#
+services.AddAWSService<IAmazonS3>(new AWSOptions 
+{
+    Credentials = new AnonymousAWSCredentials(),
+    Region = RegionEndpoint.USEast1
+});
+
+- or -
+
+services.AddAWSService<IAmazonS3>(new AWSOptions 
+{
+    Credentials = new BasicAWSCredentials(accessKeyId, secretAccessKey),
+    Region = RegionEndpoint.USEast1
+});
+
+```
+
+## Breaking Change - MapPrefix RegionEndpoint Removed
+
+The `RegionEndpoint` parameter has been removed from the `S3ServiceOptions.MapPrefix()` methods in favour of leveraging the standard AWS configuration options.  The `AddAWSService<>` method can pull configuration from a varienty of places such as the SDK store, instance profiles, configuration files, etc.  You can also set explicitly in code as in the example above.
+
+
+## Breaking Change - MapPrefix S3ClientFactory Removed
+There may be cases where different prefix mapping require different s3 client credentials.  In the previous release this could be accomplished by passing a `Func<IAmazonS3> s3ClientFactory` that would be called to create a client for the mapping on every request, and disposing the client at the end of the request.  This has been replaced by a `IAmazonS3 s3Client` parameter that will be used for the lifetime of the application.  Create an s3 client by any means supported by the AWSSDK and pass it into `MapPrefix()` 
+```c#
+var s3client1 = new AmazonS3Client(new AnonymousAWSCredentials(), RegionEndpoint.USEast1);
+var s3client2 = Configuration.GetAWSOptions().CreateServiceClient<IAmazonS3>();
+var s3client3 = Configuration.GetAWSOptions("ClientConfig3").CreateServiceClient<IAmazonS3>();
+
+services.AddImageflowS3Service(new S3ServiceOptions()
+    .MapPrefix("/path1/", s3client1, "bucket1", "", false, false)
+    .MapPrefix("/path2/", s3client2, "bucket2", "", false, false)
+    .MapPrefix("/path3/", s3client3, "bucket3", "", false, false)
+);
+```
+
+## Reference
+https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-netcore.html

src/Imageflow.Server.Storage.S3/S3Service.cs

@@ -8,12 +8,14 @@
 
 namespace Imageflow.Server.Storage.S3
 {
-    public class S3Service : IBlobProvider
+    public class S3Service : IBlobProvider, IDisposable
     {
         private readonly List<PrefixMapping> mappings = new List<PrefixMapping>();
+        private readonly IAmazonS3 s3client;
 
-        public S3Service(S3ServiceOptions options, ILogger<S3Service> logger)
+        public S3Service(S3ServiceOptions options, IAmazonS3 s3client, ILogger<S3Service> logger)
         {
+            this.s3client = s3client;
             foreach (var m in options.Mappings)
             {
                 mappings.Add(m);;
@@ -54,7 +56,7 @@ public async Task<IBlobData> Fetch(string virtualPath)
 
             try
             {
-                using var client = mapping.ClientFactory();
+                var client = mapping.S3Client ?? this.s3client;
                 var req = new Amazon.S3.Model.GetObjectRequest() { BucketName = mapping.Bucket, Key = key };
 
                 var s = await client.GetObjectAsync(req);
@@ -68,5 +70,10 @@ public async Task<IBlobData> Fetch(string virtualPath)
                 throw;
             }
         }
+
+        public void Dispose()
+        {
+            try { s3client?.Dispose(); } catch { }
+        }
     }
 }

src/Imageflow.Server.Storage.S3/S3ServiceExtensions.cs

@@ -1,3 +1,4 @@
+using Amazon.S3;
 using Imazen.Common.Storage;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -14,7 +15,8 @@ public static IServiceCollection AddImageflowS3Service(this IServiceCollection s
             services.AddSingleton<IBlobProvider>((container) =>
             {
                 var logger = container.GetRequiredService<ILogger<S3Service>>();
-                return new S3Service(options, logger);
+                var s3 = container.GetRequiredService<IAmazonS3>();
+                return new S3Service(options, s3, logger);
             });
 
             return services;

src/Imageflow.Server.Storage.S3/S3ServiceOptions.cs

@@ -8,39 +8,17 @@ namespace Imageflow.Server.Storage.S3
 {
     public class S3ServiceOptions
     {
-        private readonly AWSCredentials credentials;
         internal readonly List<PrefixMapping> Mappings = new List<PrefixMapping>();
 
-        public S3ServiceOptions()
-        {
-            credentials = new AnonymousAWSCredentials();
-        }
-
-        public S3ServiceOptions(AWSCredentials credentials)
-        {
-            this.credentials = credentials;
-        }
-
-        public S3ServiceOptions(string accessKeyId, string secretAccessKey)
-        {
-            credentials = accessKeyId == null
-                ? (AWSCredentials) new AnonymousAWSCredentials()
-                : new BasicAWSCredentials(accessKeyId, secretAccessKey);
-        }
-
-        public S3ServiceOptions MapPrefix(string prefix, RegionEndpoint region, string bucket)
-            => MapPrefix(prefix, region, bucket, "");
+        public S3ServiceOptions MapPrefix(string prefix, string bucket)
+            => MapPrefix(prefix, bucket, "");
 
-        public S3ServiceOptions MapPrefix(string prefix, RegionEndpoint region, string bucket, bool ignorePrefixCase,
+        public S3ServiceOptions MapPrefix(string prefix, string bucket, bool ignorePrefixCase,
             bool lowercaseBlobPath)
-            => MapPrefix(prefix, region, bucket, "", ignorePrefixCase, lowercaseBlobPath);
-        public S3ServiceOptions MapPrefix(string prefix, RegionEndpoint region, string bucket, string blobPrefix)
-            => MapPrefix(prefix, region, bucket, blobPrefix, false, false);
+            => MapPrefix(prefix, bucket, "", ignorePrefixCase, lowercaseBlobPath);
+        public S3ServiceOptions MapPrefix(string prefix, string bucket, string blobPrefix)
+            => MapPrefix(prefix, bucket, blobPrefix, false, false);
 
-        public S3ServiceOptions MapPrefix(string prefix, RegionEndpoint region, string bucket, string blobPrefix,
-            bool ignorePrefixCase, bool lowercaseBlobPath)
-            => MapPrefix(prefix, new AmazonS3Config() { RegionEndpoint = region }, bucket,
-                blobPrefix, ignorePrefixCase, lowercaseBlobPath);
 
         /// <summary>
         /// Maps a given prefix to a specified location within a bucket
@@ -55,12 +33,9 @@ public S3ServiceOptions MapPrefix(string prefix, RegionEndpoint region, string b
         /// (requires that actual blobs all be lowercase).</param>
         /// <returns></returns>
         /// <exception cref="ArgumentException"></exception>
-        public S3ServiceOptions MapPrefix(string prefix, AmazonS3Config s3Config, string bucket, string blobPrefix,
+        public S3ServiceOptions MapPrefix(string prefix, string bucket, string blobPrefix,
             bool ignorePrefixCase, bool lowercaseBlobPath)
-        {
-            Func<IAmazonS3> client = () => new AmazonS3Client(credentials, s3Config);
-            return MapPrefix(prefix, client, bucket, blobPrefix, ignorePrefixCase, lowercaseBlobPath);
-        }
+            => MapPrefix(prefix, null, bucket, blobPrefix, ignorePrefixCase, lowercaseBlobPath);
 
         /// <summary>
         /// Maps a given prefix to a specified location within a bucket
@@ -75,7 +50,7 @@ public S3ServiceOptions MapPrefix(string prefix, AmazonS3Config s3Config, string
         /// (requires that actual blobs all be lowercase).</param>
         /// <returns></returns>
         /// <exception cref="ArgumentException"></exception>
-        public S3ServiceOptions MapPrefix(string prefix, Func<IAmazonS3> s3ClientFactory, string bucket, string blobPrefix, bool ignorePrefixCase, bool lowercaseBlobPath)
+        public S3ServiceOptions MapPrefix(string prefix, IAmazonS3 s3Client, string bucket, string blobPrefix, bool ignorePrefixCase, bool lowercaseBlobPath)
         {
             prefix = prefix.TrimStart('/').TrimEnd('/');
             if (prefix.Length == 0)
@@ -86,11 +61,11 @@ public S3ServiceOptions MapPrefix(string prefix, Func<IAmazonS3> s3ClientFactory
             prefix = '/' + prefix + '/';
             blobPrefix = blobPrefix.Trim('/');
 
-            Mappings.Add(new PrefixMapping()
+            Mappings.Add(new PrefixMapping
             {
                 Bucket = bucket,
                 Prefix = prefix,
-                ClientFactory = s3ClientFactory,
+                S3Client = s3Client,
                 BlobPrefix = blobPrefix,
                 IgnorePrefixCase = ignorePrefixCase,
                 LowercaseBlobPath = lowercaseBlobPath

tests/Imageflow.Server.Tests/IntegrationTest.cs

@@ -218,10 +218,11 @@ public async void TestAmazonS3()
                 var hostBuilder = new HostBuilder()
                     .ConfigureServices(services =>
                     {
+                        services.AddSingleton<IAmazonS3>(new AmazonS3Client(new AnonymousAWSCredentials(), RegionEndpoint.USEast1));
                         services.AddImageflowDiskCache(new DiskCacheOptions(diskCacheDir) {AsyncWrites = false});
                         services.AddImageflowS3Service(
-                            new S3ServiceOptions(null, null)
-                                .MapPrefix("/ri/", RegionEndpoint.USEast1, "resizer-images"));
+                            new S3ServiceOptions()
+                                .MapPrefix("/ri/", "resizer-images"));
                     })
                     .ConfigureWebHost(webHost =>
                     {
@@ -268,10 +269,11 @@ public async void TestAmazonS3WithCustomClient()
                 var hostBuilder = new HostBuilder()
                     .ConfigureServices(services =>
                     {
+                        services.AddSingleton<IAmazonS3>(new AmazonS3Client(new AnonymousAWSCredentials(), RegionEndpoint.USEast1));
                         services.AddImageflowDiskCache(new DiskCacheOptions(diskCacheDir) {AsyncWrites = false});
                         services.AddImageflowS3Service(
                             new S3ServiceOptions()
-                                .MapPrefix("/ri/", () => new AmazonS3Client(new AnonymousAWSCredentials(), RegionEndpoint.USEast1), "resizer-images", "", false, false));
+                                .MapPrefix("/ri/", new AmazonS3Client(new AnonymousAWSCredentials(), RegionEndpoint.USEast1), "resizer-images", "", false, false));
                     })
                     .ConfigureWebHost(webHost =>
                     {