Stream provider (#38)

* Added CosmosDB Stream Provider * Update Readme * Fix readme * Missing CI file change * Fix CI file * Force SDK 3.0.100 * CI fix * Update test SDK * Use .MSBuild codegen package * Fix CI * Remove unecessary package CLI reference * nipt * Final updates
OrleansContrib · Nov 13, 2019 · 303c0f9 · 303c0f9
1 parent a4e8afb
commit 303c0f9
Show file tree

Hide file tree

Showing 28 changed files with 1,322 additions and 424 deletions.
diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -1,10 +1,15 @@
 pool:
-  vmImage: "ubuntu-16.04"
+  vmImage: "windows-2019"
 
 variables:
   buildConfiguration: "Release"
 
 steps:
+  - task: DotNetCoreInstaller@0
+    inputs:
+      packageType: 'sdk'
+      version: 3.0.100
+
   - task: DotNetCoreCLI@2
     inputs:
       command: "restore"

diff --git a/.vsts-release.yml b/.vsts-release.yml
@@ -1,10 +1,15 @@
 pool:
-  vmImage: "ubuntu-16.04"
+  vmImage: "windows-2019"
 
 variables:
   buildConfiguration: "Release"
 
 steps:
+  - task: DotNetCoreInstaller@0
+    inputs:
+      packageType: 'sdk'
+      version: 3.0.100
+
   - task: DotNetCoreCLI@2
     inputs:
       command: "restore"
@@ -43,6 +48,15 @@ steps:
       versioningScheme: byEnvVar
       versionEnvVar: Version
 
+  - task: DotNetCoreCLI@2
+    inputs:
+      command: pack
+      packagesToPack: "src/Orleans.Streaming.CosmosDB/*.csproj"
+      packDirectory: "$(build.artifactStagingDirectory)"
+      configuration: "$(buildConfiguration)"
+      versioningScheme: byEnvVar
+      versionEnvVar: Version
+
   - task: NuGetCommand@2
     inputs:
       command: push

diff --git a/Orleans.CosmosDB.sln b/Orleans.CosmosDB.sln
@@ -25,6 +25,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Persistence.CosmosD
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Reminders.CosmosDB", "src\Orleans.Reminders.CosmosDB\Orleans.Reminders.CosmosDB.csproj", "{E82D600B-2C44-4458-AB68-BCC25DE16631}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Orleans.Streaming.CosmosDB", "src\Orleans.Streaming.CosmosDB\Orleans.Streaming.CosmosDB.csproj", "{97AC434A-A072-44E4-B8F3-CCFD87A94F08}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -83,6 +85,18 @@ Global
 		{E82D600B-2C44-4458-AB68-BCC25DE16631}.Release|x64.Build.0 = Release|Any CPU
 		{E82D600B-2C44-4458-AB68-BCC25DE16631}.Release|x86.ActiveCfg = Release|Any CPU
 		{E82D600B-2C44-4458-AB68-BCC25DE16631}.Release|x86.Build.0 = Release|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Debug|x64.Build.0 = Debug|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Debug|x86.Build.0 = Debug|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Release|Any CPU.Build.0 = Release|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Release|x64.ActiveCfg = Release|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Release|x64.Build.0 = Release|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Release|x86.ActiveCfg = Release|Any CPU
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -92,6 +106,7 @@ Global
 		{E388AE16-3DE3-4181-AF2F-B880A78584AB} = {12BE367B-569F-4C2E-AC15-876709C119D1}
 		{1262090F-EDB2-47F1-AC69-48195CDFFBE6} = {12BE367B-569F-4C2E-AC15-876709C119D1}
 		{E82D600B-2C44-4458-AB68-BCC25DE16631} = {12BE367B-569F-4C2E-AC15-876709C119D1}
+		{97AC434A-A072-44E4-B8F3-CCFD87A94F08} = {12BE367B-569F-4C2E-AC15-876709C119D1}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9740C707-08EC-41C2-A199-A861D618296C}

diff --git a/README.md b/README.md
@@ -28,6 +28,8 @@ From Package Manager:
 
 > PS> Install-Package Orleans.Reminders.CosmosDB -prerelease
 
+> PS> Install-Package Orleans.Streaming.CosmosDB -prerelease
+
 .Net CLI:
 
 > \# dotnet add package Orleans.Clustering.CosmosDB -prerelease
@@ -36,6 +38,8 @@ From Package Manager:
 
 > \# dotnet add package Orleans.Reminders.CosmosDB -prerelease
 
+> \# dotnet add package Orleans.Streaming.CosmosDB -prerelease
+
 Paket: 
 
 > \# paket add Orleans.Clustering.CosmosDB -prerelease
@@ -44,6 +48,8 @@ Paket:
 
 > \# paket add Orleans.Reminders.CosmosDB -prerelease
 
+> \# paket add Orleans.Streaming.CosmosDB -prerelease
+
 # Configuration
 
 It is not mandatory to use all the providers at once. Just pick the one you are interested in from the samples and you should be good as they don't depend on each other.
@@ -127,5 +133,36 @@ The change will not affect existing systems. Configuring a custom `IPartitionKey
 ### Indexing
 The current indexing fork relies on CosmosDB stored procedures for lookup. As stored procedures must be executed against a specific partition, the use of custom partition key builders is not compatible with the Orleans indexing fork. 
 
+## Stream Provider
+
+To use the Stream Provider you need to register it on your `ISiloBuilder`:
+
+```csharp
+.AddCosmosDBStreaming(config => 
+    config.AddStream("<provider name>", configure =>
+    {
+        // The information on FeedCollectionInfo property is related to the database that will be monitored by the change feed
+        configure.FeedCollectionInfo = new DocumentCollectionInfo
+        {
+            Uri = new Uri("<CosmosDB URI>"),
+            MasterKey = "<CosmosDB Master Key>" ,
+            DatabaseName = "<CosmosDB Database>",
+            CollectionName = "<CosmosDB Collection>" 
+        };
+
+        // The information on LeaseCollectionInfo is related to the CosmosDB Change Feed lease collection
+        configure.LeaseCollectionInfo = new DocumentCollectionInfo
+        {
+            Uri = new Uri("<CosmosDB Change Feed Lease URI>"),
+            MasterKey = "<CosmosDB Change Feed Lease Master Key>" ,
+            DatabaseName = "<CosmosDB Change Feed Lease Database>",
+            CollectionName = "<CosmosDB Change Feed Lease Collection>" 
+        };
+    }, typeof(PartitionKeyBasedStreamMapper)))
+
+```
+
+Then on your grain, you need to implement `IAsyncObserver<Document>` in order to receive the document that has changed and published thru Cosmos DB Change Feed.
+
 # Contributions
 PRs and feedback are **very** welcome!
diff --git a/global.json b/global.json
@@ -0,0 +1,5 @@
+{
+  "sdk": {
+    "version": "3.0.100"
+  }
+}
diff --git a/src/Orleans.Clustering.CosmosDB/ClusteringExtensions.cs b/src/Orleans.Clustering.CosmosDB/ClusteringExtensions.cs
@@ -31,6 +31,27 @@ public static ISiloHostBuilder UseCosmosDBMembership(this ISiloHostBuilder build
             });
         }
 
+        public static ISiloBuilder UseCosmosDBMembership(this ISiloBuilder builder,
+           Action<CosmosDBClusteringOptions> configureOptions)
+        {
+            return builder.ConfigureServices(services => services.UseCosmosDBMembership(configureOptions));
+        }
+
+        public static ISiloBuilder UseCosmosDBMembership(this ISiloBuilder builder,
+            Action<OptionsBuilder<CosmosDBClusteringOptions>> configureOptions)
+        {
+            return builder.ConfigureServices(services => services.UseCosmosDBMembership(configureOptions));
+        }
+
+        public static ISiloBuilder UseCosmosDBMembership(this ISiloBuilder builder)
+        {
+            return builder.ConfigureServices(services =>
+            {
+                services.AddOptions<CosmosDBClusteringOptions>();
+                services.AddSingleton<IMembershipTable, CosmosDBMembershipTable>();
+            });
+        }
+
         public static IClientBuilder UseCosmosDBGatewayListProvider(this IClientBuilder builder, Action<CosmosDBGatewayOptions> configureOptions)
         {
             return builder.ConfigureServices(services => services.UseCosmosDBGatewayListProvider(configureOptions));

diff --git a/src/Orleans.Clustering.CosmosDB/Orleans.Clustering.CosmosDB.csproj b/src/Orleans.Clustering.CosmosDB/Orleans.Clustering.CosmosDB.csproj
@@ -62,13 +62,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.2.3" />
-    <PackageReference Include="Microsoft.Orleans.Core" Version="2.3.0" />
-    <PackageReference Include="Microsoft.Orleans.OrleansRuntime" Version="2.3.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Folder Include="Properties\" />
+    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.7.0" />
+    <PackageReference Include="Microsoft.Orleans.Core" Version="3.0.0" />
+    <PackageReference Include="Microsoft.Orleans.OrleansRuntime" Version="3.0.0" />
   </ItemGroup>
 
 </Project>
diff --git a/src/Orleans.Persistence.CosmosDB/Orleans.Persistence.CosmosDB.csproj b/src/Orleans.Persistence.CosmosDB/Orleans.Persistence.CosmosDB.csproj
@@ -38,10 +38,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.2.3" />
-    <PackageReference Include="Microsoft.Orleans.Core" Version="2.3.0" />
-    <PackageReference Include="Microsoft.Orleans.OrleansRuntime" Version="2.3.0" />
-    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.2" />
+    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.7.0" />
+    <PackageReference Include="Microsoft.Orleans.Core" Version="3.0.0" />
+    <PackageReference Include="Microsoft.Orleans.OrleansRuntime" Version="3.0.0" />
+    <PackageReference Include="System.Threading.Tasks.Extensions" Version="4.5.3" />
   </ItemGroup>
 
 </Project>
diff --git a/src/Orleans.Persistence.CosmosDB/StorageExtensions.cs b/src/Orleans.Persistence.CosmosDB/StorageExtensions.cs
@@ -14,6 +14,123 @@ namespace Orleans.Hosting
 {
     public static class StorageExtensions
     {
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage as the default grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorageAsDefault<TPartitionKeyProvider>(this ISiloBuilder builder, Action<CosmosDBStorageOptions> configureOptions) where TPartitionKeyProvider : class, IPartitionKeyProvider
+        {
+            return builder.AddCosmosDBGrainStorage<TPartitionKeyProvider>(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage for grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorage<TPartitionKeyProvider>(this ISiloBuilder builder, string name, Action<CosmosDBStorageOptions> configureOptions) where TPartitionKeyProvider : class, IPartitionKeyProvider
+        {
+            return builder.ConfigureServices(services =>
+            {
+                services.TryAddSingleton<IPartitionKeyProvider, TPartitionKeyProvider>();
+                services.AddCosmosDBGrainStorage(name, configureOptions);
+            });
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage as the default grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorageAsDefault(this ISiloBuilder builder, Action<CosmosDBStorageOptions> configureOptions, Type customPartitionKeyProviderType)
+        {
+            return builder.AddCosmosDBGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions, customPartitionKeyProviderType);
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage for grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorage(this ISiloBuilder builder, string name, Action<CosmosDBStorageOptions> configureOptions, Type customPartitionKeyProviderType)
+        {
+            return builder.ConfigureServices(services =>
+            {
+                if (customPartitionKeyProviderType != null)
+                {
+                    services.TryAddSingleton(typeof(IPartitionKeyProvider), customPartitionKeyProviderType);
+                }
+                services.AddCosmosDBGrainStorage(name, configureOptions);
+            });
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage as the default grain storage.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorageAsDefault(this ISiloBuilder builder, Action<CosmosDBStorageOptions> configureOptions)
+        {
+            return builder.AddCosmosDBGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage for grain storage.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorage(this ISiloBuilder builder, string name, Action<CosmosDBStorageOptions> configureOptions)
+        {
+            return builder.ConfigureServices(services => services.AddCosmosDBGrainStorage(name, configureOptions));
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage as the default grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorageAsDefault<TPartitionKeyProvider>(this ISiloBuilder builder, Action<OptionsBuilder<CosmosDBStorageOptions>> configureOptions = null) where TPartitionKeyProvider : class, IPartitionKeyProvider
+        {
+            return builder.AddCosmosDBGrainStorage<TPartitionKeyProvider>(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage for grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorage<TPartitionKeyProvider>(this ISiloBuilder builder, string name, Action<OptionsBuilder<CosmosDBStorageOptions>> configureOptions = null) where TPartitionKeyProvider : class, IPartitionKeyProvider
+        {
+            return builder.ConfigureServices(services =>
+            {
+                services.TryAddSingleton<IPartitionKeyProvider, TPartitionKeyProvider>();
+                services.AddCosmosDBGrainStorage(name, configureOptions);
+            });
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage as the default grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorageAsDefault(this ISiloBuilder builder, Type customPartitionKeyProviderType, Action<OptionsBuilder<CosmosDBStorageOptions>> configureOptions = null)
+        {
+            return builder.AddCosmosDBGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, customPartitionKeyProviderType, configureOptions);
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage for grain storage using a custom Partition Key Provider.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorage(this ISiloBuilder builder, string name, Type customPartitionKeyProviderType, Action<OptionsBuilder<CosmosDBStorageOptions>> configureOptions = null)
+        {
+            return builder.ConfigureServices(services =>
+            {
+                if (customPartitionKeyProviderType != null)
+                {
+                    services.TryAddSingleton(typeof(IPartitionKeyProvider), customPartitionKeyProviderType);
+                }
+                services.AddCosmosDBGrainStorage(name, configureOptions);
+            });
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage as the default grain storage.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorageAsDefault(this ISiloBuilder builder, Action<OptionsBuilder<CosmosDBStorageOptions>> configureOptions = null)
+        {
+            return builder.AddCosmosDBGrainStorage(ProviderConstants.DEFAULT_STORAGE_PROVIDER_NAME, configureOptions);
+        }
+
+        /// <summary>
+        /// Configure silo to use Azure CosmosDB storage for grain storage.
+        /// </summary>
+        public static ISiloBuilder AddCosmosDBGrainStorage(this ISiloBuilder builder, string name, Action<OptionsBuilder<CosmosDBStorageOptions>> configureOptions = null)
+        {
+            return builder.ConfigureServices(services => services.AddCosmosDBGrainStorage(name, configureOptions));
+        }
 
         /// <summary>
         /// Configure silo to use Azure CosmosDB storage as the default grain storage using a custom Partition Key Provider.

diff --git a/src/Orleans.Reminders.CosmosDB/Orleans.Reminders.CosmosDB.csproj b/src/Orleans.Reminders.CosmosDB/Orleans.Reminders.CosmosDB.csproj
@@ -58,8 +58,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.2.3" />
-    <PackageReference Include="Microsoft.Orleans.OrleansRuntime" Version="2.3.0" />
+    <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.7.0" />
+    <PackageReference Include="Microsoft.Orleans.OrleansRuntime" Version="3.0.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -74,8 +74,4 @@
     </None>
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Properties\" />
-  </ItemGroup>
-
 </Project>