From 2d531b2c3db25dc8ae1c73d25c67d99f27d6cb23 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Thu, 14 Nov 2024 16:03:58 +0000 Subject: [PATCH 1/9] #1097 - Initial commit to add gremlin graph support to cosmos db builder. --- src/Farmer/Arm/DocumentDb.fs | 107 ++++++++++++++++--------- src/Farmer/Builders/Builders.Cosmos.fs | 12 ++- 2 files changed, 80 insertions(+), 39 deletions(-) diff --git a/src/Farmer/Arm/DocumentDb.fs b/src/Farmer/Arm/DocumentDb.fs index dd47d68a1..59cd42d92 100644 --- a/src/Farmer/Arm/DocumentDb.fs +++ b/src/Farmer/Arm/DocumentDb.fs @@ -16,9 +16,16 @@ let mongoDatabases = let databaseAccounts = ResourceType("Microsoft.DocumentDb/databaseAccounts", "2021-04-15") +let gremlinDatabases = + ResourceType("Microsoft.DocumentDb/databaseAccounts/gremlinDatabases", "2022-05-15") + +let graphs = + ResourceType("Microsoft.DocumentDb/databaseAccounts/gremlinDatabases/graphs", "2022-05-15") + type DatabaseKind = | Document | Mongo + | Gremlin module DatabaseAccounts = module SqlDatabases = @@ -26,6 +33,7 @@ module DatabaseAccounts = Name: ResourceName Account: ResourceName Database: ResourceName + Kind: DatabaseKind PartitionKey: {| Paths: string list Kind: IndexKind @@ -45,44 +53,56 @@ module DatabaseAccounts = interface IArmResource with member this.ResourceId = - containers.resourceId (this.Account / this.Database / this.Name) + match this.Kind with + | Gremlin -> graphs.resourceId (this.Account / this.Database / this.Name) + | _ -> containers.resourceId (this.Account / this.Database / this.Name) - member this.JsonModel = {| - containers.Create( - this.Account / this.Database / this.Name, - dependsOn = [ sqlDatabases.resourceId (this.Account, this.Database) ] - ) with - properties = {| - resource = {| - id = this.Name.Value - partitionKey = {| - paths = this.PartitionKey.Paths - kind = string this.PartitionKey.Kind - |} - uniqueKeyPolicy = {| - uniqueKeys = - this.UniqueKeyPolicy.UniqueKeys |> Set.map (fun k -> {| paths = k.Paths |}) - |} - indexingPolicy = {| - indexingMode = "consistent" - includedPaths = - this.IndexingPolicy.IncludedPaths - |> List.map (fun p -> {| - path = p.Path - indexes = - p.Indexes - |> List.map (fun (dataType, kind) -> {| - kind = string kind - dataType = dataType.ToString().ToLower() - precision = -1 - |}) - |}) - excludedPaths = - this.IndexingPolicy.ExcludedPaths |> List.map (fun p -> {| path = p |}) + member this.JsonModel = + let resourceType = + match this.Kind with + | Gremlin -> graphs + | _ -> containers + + {| + resourceType.Create( + this.Account / this.Database / this.Name, + dependsOn = [ + match this.Kind with + | Gremlin -> gremlinDatabases.resourceId (this.Account, this.Database) + | _ -> sqlDatabases.resourceId (this.Account, this.Database) + ] + ) with + properties = {| + resource = {| + id = this.Name.Value + partitionKey = {| + paths = this.PartitionKey.Paths + kind = string this.PartitionKey.Kind + |} + uniqueKeyPolicy = {| + uniqueKeys = + this.UniqueKeyPolicy.UniqueKeys |> Set.map (fun k -> {| paths = k.Paths |}) + |} + indexingPolicy = {| + indexingMode = "consistent" + includedPaths = + this.IndexingPolicy.IncludedPaths + |> List.map (fun p -> {| + path = p.Path + indexes = + p.Indexes + |> List.map (fun (dataType, kind) -> {| + kind = string kind + dataType = dataType.ToString().ToLower() + precision = -1 + |}) + |}) + excludedPaths = + this.IndexingPolicy.ExcludedPaths |> List.map (fun p -> {| path = p |}) + |} |} |} - |} - |} + |} type SqlDatabase = { Name: ResourceName @@ -92,13 +112,17 @@ module DatabaseAccounts = } with interface IArmResource with - member this.ResourceId = sqlDatabases.resourceId (this.Account / this.Name) + member this.ResourceId = + match this.Kind with + | Gremlin -> gremlinDatabases.resourceId (this.Account / this.Name) + | _ -> sqlDatabases.resourceId (this.Account / this.Name) member this.JsonModel = let resource = match this.Kind with | Document -> sqlDatabases | Mongo -> mongoDatabases + | Gremlin -> gremlinDatabases {| resource.Create(this.Account / this.Name, dependsOn = [ databaseAccounts.resourceId this.Account ]) with @@ -176,6 +200,7 @@ type DatabaseAccount = { match this.Kind with | Document -> "GlobalDocumentDB" | Mongo -> "MongoDB" + | Gremlin -> "GlobalDocumentDB" properties = {| consistencyPolicy = {| @@ -205,10 +230,16 @@ type DatabaseAccount = { publicNetworkAccess = string this.PublicNetworkAccess enableFreeTier = this.FreeTier capabilities = - if this.Serverless = Enabled then - box [ {| name = "EnableServerless" |} ] + if this.Serverless = Enabled || this.Kind = Gremlin then + box [ + if this.Serverless = Enabled then + {| name = "EnableServerless" |} + if this.Kind = Gremlin then + {| name = "EnableGremlin" |} + ] else null + |} |> box |} \ No newline at end of file diff --git a/src/Farmer/Builders/Builders.Cosmos.fs b/src/Farmer/Builders/Builders.Cosmos.fs index 6b1b9c101..e747142db 100644 --- a/src/Farmer/Builders/Builders.Cosmos.fs +++ b/src/Farmer/Builders/Builders.Cosmos.fs @@ -2,8 +2,8 @@ module Farmer.Builders.CosmosDb open Farmer +open Farmer.Arm open Farmer.CosmosDb -open Farmer.Arm.DocumentDb open DatabaseAccounts open SqlDatabases @@ -62,6 +62,7 @@ type CosmosDbContainerConfig = { Indexes: (string * (IndexDataType * IndexKind) list) list UniqueKeys: Set ExcludedPaths: string list + Kind: DatabaseKind } type CosmosDbConfig = { @@ -138,6 +139,7 @@ type CosmosDbConfig = { Name = container.Name Account = this.AccountResourceId.Name Database = this.DbName + Kind = container.Kind PartitionKey = {| Paths = fst container.PartitionKey Kind = snd container.PartitionKey @@ -160,6 +162,7 @@ type CosmosDbConfig = { type CosmosDbContainerBuilder() = member _.Yield _ = { Name = ResourceName "" + Kind = DatabaseKind.Document PartitionKey = [], Hash Indexes = [] UniqueKeys = Set.empty @@ -186,6 +189,13 @@ type CosmosDbContainerBuilder() = [] member _.Name(state: CosmosDbContainerConfig, name) = { state with Name = ResourceName name } + /// Sets the container kind of the container. + [] + member _.Graph(state: CosmosDbContainerConfig) = { + state with + Kind = DatabaseKind.Gremlin + } + /// Sets the partition key of the container. [] member _.PartitionKey(state: CosmosDbContainerConfig, partitions, indexKind) = { From 96575f1241478440f05a8f15f804ffbddc079546 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Tue, 19 Nov 2024 11:35:01 +0000 Subject: [PATCH 2/9] #1097 - Rename SqlDatabases Module to Containers. --- src/Farmer/Arm/DocumentDb.fs | 2 +- src/Farmer/Builders/Builders.Cosmos.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Farmer/Arm/DocumentDb.fs b/src/Farmer/Arm/DocumentDb.fs index 59cd42d92..d9adbbd9c 100644 --- a/src/Farmer/Arm/DocumentDb.fs +++ b/src/Farmer/Arm/DocumentDb.fs @@ -28,7 +28,7 @@ type DatabaseKind = | Gremlin module DatabaseAccounts = - module SqlDatabases = + module Containers = type Container = { Name: ResourceName Account: ResourceName diff --git a/src/Farmer/Builders/Builders.Cosmos.fs b/src/Farmer/Builders/Builders.Cosmos.fs index e747142db..1d0df1461 100644 --- a/src/Farmer/Builders/Builders.Cosmos.fs +++ b/src/Farmer/Builders/Builders.Cosmos.fs @@ -5,7 +5,7 @@ open Farmer open Farmer.Arm open Farmer.CosmosDb open DatabaseAccounts -open SqlDatabases +open Containers type KeyType = | PrimaryKey From a9d7e5075ba415667af1b5f409d27da7e99ffa70 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Thu, 21 Nov 2024 13:28:58 +0000 Subject: [PATCH 3/9] #1097 - Fix issue that would prevent the graph database from being provisioned. --- src/Farmer/Arm/DocumentDb.fs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Farmer/Arm/DocumentDb.fs b/src/Farmer/Arm/DocumentDb.fs index d9adbbd9c..a1cf22ce1 100644 --- a/src/Farmer/Arm/DocumentDb.fs +++ b/src/Farmer/Arm/DocumentDb.fs @@ -218,14 +218,16 @@ type DatabaseAccount = { enableAutomaticFailover = this.EnableAutomaticFailover |> Option.toNullable enableMultipleWriteLocations = this.EnableMultipleWriteLocations |> Option.toNullable locations = - match this.FailoverLocations, this.Serverless with - | [], Enabled -> + // Locations has to be specified for the account to be gremlin enabled. + // Otherwise graph database provisioning fails. + match this.FailoverLocations, (this.Serverless = Enabled || this.Kind = Gremlin) with + | [], true -> box [ {| locationName = this.Location.ArmValue |} ] - | [], Disabled -> null + | [], false -> null | locations, _ -> box locations publicNetworkAccess = string this.PublicNetworkAccess enableFreeTier = this.FreeTier From 93efdab627f3538e4dc5c2a9800ad1e4597dbcb9 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Thu, 21 Nov 2024 13:30:01 +0000 Subject: [PATCH 4/9] #1097 - Add some tests --- src/Tests/Cosmos.fs | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/Tests/Cosmos.fs b/src/Tests/Cosmos.fs index 103a59e9d..4f0271eef 100644 --- a/src/Tests/Cosmos.fs +++ b/src/Tests/Cosmos.fs @@ -146,6 +146,15 @@ let tests = Expect.equal db.Kind Mongo "" } + + test "gremlin" { + let db = cosmosDb { + name "test" + kind Gremlin + } + + Expect.equal db.Kind Gremlin "" + } ] test "Correctly serializes to JSON" { @@ -153,6 +162,27 @@ let tests = t.Template |> Writer.toJson |> ignore } + + test "Correctly serializes gremlin db and container to JSON" { + let t = arm { + add_resource ( + cosmosDb { + name "test" + kind Gremlin + + add_containers [ + cosmosContainer { + name "myContainer" + partition_key [ "pk" ] CosmosDb.Hash + gremlin_graph + } + ] + } + ) + } + + t.Template |> Writer.toJson |> ignore + } test "Creates connection string and keys with resource groups" { let conn = CosmosDb From 9749ee34da5b8c1da50c01a97dbaa46acf788814 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Thu, 21 Nov 2024 13:49:26 +0000 Subject: [PATCH 5/9] #1097 - Prevent unsupported cosmos container types from being provisioned under mismatched database accounts. --- src/Farmer/Builders/Builders.Cosmos.fs | 30 ++++++++++++++++++++++---- src/Tests/Cosmos.fs | 24 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/Farmer/Builders/Builders.Cosmos.fs b/src/Farmer/Builders/Builders.Cosmos.fs index 1d0df1461..65ec96f59 100644 --- a/src/Farmer/Builders/Builders.Cosmos.fs +++ b/src/Farmer/Builders/Builders.Cosmos.fs @@ -1,6 +1,7 @@ [] module Farmer.Builders.CosmosDb +open System open Farmer open Farmer.Arm open Farmer.CosmosDb @@ -306,12 +307,33 @@ type CosmosDbBuilder() = [] member _.StorageKind(state: CosmosDbConfig, kind) = { state with Kind = kind } + static member ValidateContainers(state: CosmosDbConfig, containers: CosmosDbContainerConfig list) = + let validateContainerAndAccountConfig (container: CosmosDbContainerConfig, accountKind: DatabaseKind) = + if container.Kind = accountKind then + Ok container + else + Error $"Container {container.Name.Value} must be of {state.Kind} kind" + + containers + |> List.map (fun container -> validateContainerAndAccountConfig (container, state.Kind)) + /// Adds a list of containers to the database. [] - member _.AddContainers(state: CosmosDbConfig, containers) = { - state with - Containers = state.Containers @ containers - } + member _.AddContainers(state: CosmosDbConfig, containers) = + let errors = + CosmosDbBuilder.ValidateContainers(state, containers) + |> List.choose (fun r -> + match r with + | Error e -> Some(e) + | Ok _ -> None) + + if errors.Length > 0 then + errors |> String.concat Environment.NewLine |> failwith + + { + state with + Containers = state.Containers @ containers + } /// Enables public network access [] diff --git a/src/Tests/Cosmos.fs b/src/Tests/Cosmos.fs index 4f0271eef..ee6b9dfcb 100644 --- a/src/Tests/Cosmos.fs +++ b/src/Tests/Cosmos.fs @@ -183,6 +183,30 @@ let tests = t.Template |> Writer.toJson |> ignore } + + test "Gremlin graph container cannot be created in mongo account" { + let testCase () = + let t = arm { + add_resource ( + cosmosDb { + name "test" + kind Mongo + + add_containers [ + cosmosContainer { + name "myContainer" + partition_key [ "pk" ] CosmosDb.Hash + gremlin_graph + } + ] + } + ) + } + + t.Template |> Writer.toJson |> ignore + + Expect.throws (fun _ -> testCase ()) "Container \"myContainer\" must be of Mongo kind" + } test "Creates connection string and keys with resource groups" { let conn = CosmosDb From 41316e81f4c6c6f5dc7832c36434413cdce68761 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Fri, 22 Nov 2024 14:59:23 +0000 Subject: [PATCH 6/9] #1097 - Validate the account and container kind in the Run method of the builder. --- src/Farmer/Builders/Builders.Cosmos.fs | 42 ++++++++++++++------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Farmer/Builders/Builders.Cosmos.fs b/src/Farmer/Builders/Builders.Cosmos.fs index 65ec96f59..7730d6ac3 100644 --- a/src/Farmer/Builders/Builders.Cosmos.fs +++ b/src/Farmer/Builders/Builders.Cosmos.fs @@ -250,6 +250,28 @@ type CosmosDbBuilder() = Kind = DatabaseKind.Document } + static member ValidateContainers(state: CosmosDbConfig) = + let validateContainerAndAccountConfig (container: CosmosDbContainerConfig, accountKind: DatabaseKind) = + if container.Kind = accountKind then + Ok container + else + Error $"Container {container.Name.Value} must be of {state.Kind} kind" + + state.Containers + |> List.map (fun container -> validateContainerAndAccountConfig (container, state.Kind)) + + member _.Run state = + let errors = + CosmosDbBuilder.ValidateContainers(state) + |> List.choose (fun r -> + match r with + | Error e -> Some(e) + | Ok _ -> None) + + if errors.Length > 0 then + errors |> String.concat Environment.NewLine |> raiseFarmer + state + /// Sets the name of the CosmosDB server. [] member _.AccountName(state: CosmosDbConfig, accountName: ResourceName) = { @@ -307,29 +329,9 @@ type CosmosDbBuilder() = [] member _.StorageKind(state: CosmosDbConfig, kind) = { state with Kind = kind } - static member ValidateContainers(state: CosmosDbConfig, containers: CosmosDbContainerConfig list) = - let validateContainerAndAccountConfig (container: CosmosDbContainerConfig, accountKind: DatabaseKind) = - if container.Kind = accountKind then - Ok container - else - Error $"Container {container.Name.Value} must be of {state.Kind} kind" - - containers - |> List.map (fun container -> validateContainerAndAccountConfig (container, state.Kind)) - /// Adds a list of containers to the database. [] member _.AddContainers(state: CosmosDbConfig, containers) = - let errors = - CosmosDbBuilder.ValidateContainers(state, containers) - |> List.choose (fun r -> - match r with - | Error e -> Some(e) - | Ok _ -> None) - - if errors.Length > 0 then - errors |> String.concat Environment.NewLine |> failwith - { state with Containers = state.Containers @ containers From 7bbecb276aa73da820ee2e6082f1aaa9acd356a6 Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Mon, 25 Nov 2024 15:13:08 +0000 Subject: [PATCH 7/9] #1097 - Format --- src/Farmer/Builders/Builders.Cosmos.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Farmer/Builders/Builders.Cosmos.fs b/src/Farmer/Builders/Builders.Cosmos.fs index 7730d6ac3..e55bb88f4 100644 --- a/src/Farmer/Builders/Builders.Cosmos.fs +++ b/src/Farmer/Builders/Builders.Cosmos.fs @@ -270,6 +270,7 @@ type CosmosDbBuilder() = if errors.Length > 0 then errors |> String.concat Environment.NewLine |> raiseFarmer + state /// Sets the name of the CosmosDB server. @@ -331,11 +332,10 @@ type CosmosDbBuilder() = /// Adds a list of containers to the database. [] - member _.AddContainers(state: CosmosDbConfig, containers) = - { - state with - Containers = state.Containers @ containers - } + member _.AddContainers(state: CosmosDbConfig, containers) = { + state with + Containers = state.Containers @ containers + } /// Enables public network access [] From 9a1f4e3db9005010f2f14e9d0f345b0d79e659fc Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Wed, 4 Dec 2024 15:09:38 +0000 Subject: [PATCH 8/9] #1097 - Update documentation showing how to create a gremlin account. --- .../api-overview/resources/cosmos-db.md | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/content/api-overview/resources/cosmos-db.md b/docs/content/api-overview/resources/cosmos-db.md index 8be991b38..4622f36cf 100644 --- a/docs/content/api-overview/resources/cosmos-db.md +++ b/docs/content/api-overview/resources/cosmos-db.md @@ -12,35 +12,39 @@ The CosmosDb package containers two builders, used to create *databases* and *co * CosmosDB SQL (`Microsoft.DocumentDB/databaseAccounts/sqlDatabases`) * CosmosDB MongoDB (`Microsoft.DocumentDB/databaseAccounts/mongodbDatabases`) * CosmosDB SQL Container (`Microsoft.DocumentDb/databaseAccounts/sqlDatabases/containers`) +* CosmosDB Graph databases (`Microsoft.DocumentDb/databaseAccounts/gremlinDatabases`) +* CosmosDB Graph containers (`Microsoft.DocumentDb/databaseAccounts/gremlinDatabases/graphs`) -> There is currently only support for document databases (the so-called "SQL API"), with support for Gremlin, Table and Cassandra data models planned. +> There is currently support for document databases (the so-called "SQL API") and Gremlin graphs. Support for Table and Cassandra data models planned. #### Cosmos DB Builder The CosmosDB builder abstracts the idea of an account and database into one. If you wish to "re-use" an already-created Cosmos DB account, use the `link_to_account` keyword - no account will be created and the database will be attached to the existing one. -| Applies To | Keyword | Purpose | -|-|-|-| -| Database | name | Sets the name of the database. | -| Database | link_to_account | Instructs Farmer to link this database to an existing Cosmos DB account rather than creating a new one. | -| Database | throughput | Sets the throughput with either "provisioned throughput" or "serverless". | -| Database | add_containers | Adds a list of containers to the database. | -| Account | account_name | Sets the name of the CosmosDB account. | -| Account | api (not yet implemented) | Sets the API and data model to use -- currently defaults to "Core (SQL)". | -| Account | enable_public_network_access | Enables public network access for the account. | +| Applies To | Keyword | Purpose | +|-|-------------------------------|-| +| Database | name | Sets the name of the database. | +| Database | link_to_account | Instructs Farmer to link this database to an existing Cosmos DB account rather than creating a new one. | +| Database | throughput | Sets the throughput with either "provisioned throughput" or "serverless". | +| Database | add_containers | Adds a list of containers to the database. | +| Account | account_name | Sets the name of the CosmosDB account. | +| Account | kind | Sets the API and data model to use -- currently defaults to "Core (SQL)". | +| Account | enable_public_network_access | Enables public network access for the account. | | Account | disable_public_network_access | Disables public network access for the account. | -| Account | consistency_policy | Sets the consistency policy of the database. | -| Account | failover_policy | Sets the failover policy of the database. | -| Account | free_tier | Registers this server with the free pricing tier, if supported and allowed by Azure. | +| Account | consistency_policy | Sets the consistency policy of the database. | +| Account | failover_policy | Sets the failover policy of the database. | +| Account | free_tier | Registers this server with the free pricing tier, if supported and allowed by Azure. | #### Cosmos Container Builder The container builder allows you to create and configure a specific container that is attached to a cosmos database. -| Keyword | Purpose | -|-|-| -| name | Sets the name of the container. | -| partition_key | Sets the partition key of the container. | -| add_index | Adds an index to the container. | -| exclude_path | Excludes a path from the container index. | +| Keyword | Purpose | +|-|--------------------------------------------------------------------------------------------| +| name | Sets the name of the container. | +| partition_key | Sets the partition key of the container. | +| add_index | Adds an index to the container. | +| exclude_path | Excludes a path from the container index. | +| gremlin_graph | Marks the container as graph (must be used with an account of `kind DatabaseKind.Gremlin`) | + #### Example ```fsharp @@ -53,12 +57,14 @@ let myCosmosDb = cosmosDb { throughput 400 // or throughput Serverless failover_policy CosmosDb.NoFailover consistency_policy (CosmosDb.BoundedStaleness(500, 1000)) + //kind DatabaseKind.Gremlin //Create a gremlin enabled account add_containers [ cosmosContainer { name "myContainer" partition_key [ "/id" ] CosmosDb.Hash add_index "/path" [ CosmosDb.Number, CosmosDb.Hash ] exclude_path "/excluded/*" + //gremlin_graph //Mark this container to be a graph } ] } From ba54df0f37283c3472b54990107442d18a72828d Mon Sep 17 00:00:00 2001 From: "C5ALLIANCE\\Michael.Wade" Date: Wed, 4 Dec 2024 15:16:40 +0000 Subject: [PATCH 9/9] #1097 - Update release notes --- RELEASE_NOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d59bc1abf..ece69ae4f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,9 @@ Release Notes ============= +## 1.9.6 +* CosmosDB: Add support for Gremlin Graphs + ## 1.9.5 * Fix an issue with Access Policies that allows non-string sequences to be supplied for user / group lookups. * Virtual Network: Specify the route table for a subnet.