Skip to content

Commit 135d76a

Browse files
jshlbrdjshlbrd
authored
docs(examples): Add DynamoDB Telephone Example (#164)
* docs(examples): Add DynamoDB Telephone Example * docs: Rename Resources * docs: Examples README
1 parent 6ca3ae8 commit 135d76a

File tree

15 files changed

+417
-0
lines changed

15 files changed

+417
-0
lines changed

‎examples/terraform/aws/README.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,59 @@ flowchart LR
8282
end
8383
```
8484

85+
## Telephone
86+
87+
Deploys a data pipeline that implements a "telephone" pattern by sharing data as context between multiple Lambda functions using a DynamoDB table. This pattern can be used to enrich events across unique data sources.
88+
89+
```mermaid
90+
91+
flowchart LR
92+
%% resources
93+
md_kinesis([Device Management
94+
Kinesis Data Stream])
95+
edr_kinesis([EDR Kinesis Data Stream])
96+
idp_kinesis([IdP Kinesis Data Stream])
97+
dynamodb([DynamoDB Table])
98+
99+
edrEnrichmentHandler[[Handler]]
100+
edrEnrichmentTransforms[Transforms]
101+
102+
edrTransformHandler[[Handler]]
103+
edrTransformTransforms[Transforms]
104+
105+
idpEnrichmentHandler[[Handler]]
106+
idpEnrichmentTransforms[Transforms]
107+
108+
mdEnrichmentHandler[[Handler]]
109+
mdEnrichmentTransforms[Transforms]
110+
111+
%% connections
112+
edr_kinesis --> edrEnrichmentHandler
113+
subgraph Substation EDR Enrichment Node
114+
edrEnrichmentHandler --> edrEnrichmentTransforms
115+
end
116+
117+
edr_kinesis --> edrTransformHandler
118+
subgraph Substation EDR Transform Node
119+
edrTransformHandler --> edrTransformTransforms
120+
end
121+
122+
idp_kinesis --> idpEnrichmentHandler
123+
subgraph Substation IdP Enrichment Node
124+
idpEnrichmentHandler --> idpEnrichmentTransforms
125+
end
126+
127+
md_kinesis --> mdEnrichmentHandler
128+
subgraph Substation Dvc Mgmt Enrichment Node
129+
mdEnrichmentHandler --> mdEnrichmentTransforms
130+
end
131+
132+
edrEnrichmentTransforms --- dynamodb
133+
edrTransformTransforms --- dynamodb
134+
idpEnrichmentTransforms --- dynamodb
135+
mdEnrichmentTransforms --- dynamodb
136+
```
137+
85138
# Firehose
86139

87140
## Data Transform

‎examples/terraform/aws/dynamodb/telephone/config/const.libsonnet

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
local sub = import '../../../../../../build/config/substation.libsonnet';
2+
3+
{
4+
kv_store: sub.kv_store.aws_dynamodb({
5+
table_name: 'substation',
6+
attributes: { partition_key: 'PK', sort_key: 'SK', ttl: 'TTL', value: 'cache' },
7+
}),
8+
}

‎examples/terraform/aws/dynamodb/telephone/config/dvc_mgmt_enrichment/config.jsonnet

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
local sub = import '../../../../../../../build/config/substation.libsonnet';
2+
local const = import '../const.libsonnet';
3+
4+
{
5+
concurrency: 1,
6+
transforms: [
7+
// Puts the user's metadata into the KV store indexed by the host name.
8+
sub.tf.enrich.kv_store.set({ obj: { src: 'host.name', trg: 'user' }, prefix: 'md_user', kv_store: const.kv_store }),
9+
],
10+
}

‎examples/terraform/aws/dynamodb/telephone/config/edr_enrichment/config.jsonnet

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
local sub = import '../../../../../../../build/config/substation.libsonnet';
2+
local const = import '../const.libsonnet';
3+
4+
{
5+
concurrency: 1,
6+
transforms: [
7+
// If the host metadata contains the host name, then it's put into the KV store
8+
// indexed by the host ID.
9+
sub.tf.meta.switch({ cases: [
10+
{
11+
condition: sub.cnd.all([
12+
sub.cnd.num.len.gt({ obj: { src: 'host.name' }, value: 0 }),
13+
]),
14+
transform: sub.tf.enrich.kv_store.set({ obj: { src: 'host.id', trg: 'host' }, prefix: 'edr_host', kv_store: const.kv_store }),
15+
},
16+
] }),
17+
],
18+
}

‎examples/terraform/aws/dynamodb/telephone/config/edr_transform/config.jsonnet

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
local sub = import '../../../../../../../build/config/substation.libsonnet';
2+
local const = import '../const.libsonnet';
3+
4+
// cnd_copy is a helper function for copying values that are not null.
5+
local cnd_copy(source, target) = sub.pattern.tf.conditional(
6+
condition=sub.cnd.num.len.gt({ obj: { src: source }, value: 0 }),
7+
transform=sub.tf.object.copy({ obj: { src: source, trg: target } }),
8+
);
9+
10+
{
11+
concurrency: 1,
12+
transforms: [
13+
// The value from the KV store can be null, so the result is hidden in metadata and checked before
14+
// copying it into the message data. Many of these values are supersets of each other, so values are
15+
// overwritten if they exist. If any source key is missing, the transform is skipped.
16+
sub.tf.enrich.kv_store.get({ obj: { src: 'host.id', trg: 'meta edr_host' }, prefix: 'edr_host', kv_store: const.kv_store }),
17+
cnd_copy(source='meta edr_host', target='host'),
18+
sub.tf.enrich.kv_store.get({ obj: { src: 'host.name', trg: 'meta md_user' }, prefix: 'md_user', kv_store: const.kv_store }),
19+
cnd_copy(source='meta md_user', target='user'),
20+
sub.tf.enrich.kv_store.get({ obj: { src: 'user.email', trg: 'meta idp_user' }, prefix: 'idp_user', kv_store: const.kv_store }),
21+
cnd_copy(source='meta idp_user', target='user'),
22+
sub.tf.send.stdout(),
23+
],
24+
}

‎examples/terraform/aws/dynamodb/telephone/config/idp_enrichment/config.jsonnet

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
local sub = import '../../../../../../../build/config/substation.libsonnet';
2+
local const = import '../const.libsonnet';
3+
4+
{
5+
concurrency: 1,
6+
transforms: [
7+
// The user's status is determined to be inactive if there is a successful deletion event.
8+
// Any other successful authentication event will set the user's status to active.
9+
//
10+
// In production deployments, additional filtering should be used to reduce the number of
11+
// queries made to the KV store.
12+
sub.tf.meta.switch({ cases: [
13+
{
14+
condition: sub.cnd.all([
15+
sub.cnd.str.eq({ object: { source_key: 'event.category' }, value: 'authentication' }),
16+
sub.cnd.str.eq({ object: { source_key: 'event.type' }, value: 'deletion' }),
17+
sub.cnd.str.eq({ object: { source_key: 'event.outcome' }, value: 'success' }),
18+
]),
19+
transform: sub.tf.object.insert({ object: { target_key: 'user.status.-1' }, value: 'idp_inactive' }),
20+
},
21+
{
22+
condition: sub.cnd.all([
23+
sub.cnd.str.eq({ object: { source_key: 'event.outcome' }, value: 'success' }),
24+
]),
25+
transform: sub.tf.object.insert({ object: { target_key: 'user.status.-1' }, value: 'idp_active' }),
26+
},
27+
] }),
28+
// Puts the user's metadata into the KV store indexed by their email address.
29+
sub.tf.enrich.kv_store.set({ obj: { src: 'user.email', trg: 'user' }, prefix: 'idp_user', kv_store: const.kv_store }),
30+
],
31+
}

‎examples/terraform/aws/dynamodb/telephone/dvc_mgmt_data.jsonl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"host":{"name":"Alice's MacBook Pro"},"user":{"email":"[email protected]"}}
2+
{"host":{"name":"Bob's MacBook Pro"},"user":{"email":"[email protected]"}}

‎examples/terraform/aws/dynamodb/telephone/edr_data.jsonl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"host":{"id":"eb67b0b6a1d04086b75ee38d02018a10","name":"Alice's MacBook Pro"}}
2+
{"event":{"category":"network","type":"connection"},"host":{"id":"eb67b0b6a1d04086b75ee38d02018a10"},"process":{"name":"Spotify","pid":"d3a6c0b9d3751559f206e12fb1b8f226"},"server":{"ip":"35.186.224.39","port":443}}

‎examples/terraform/aws/dynamodb/telephone/idp_data.jsonl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"event":{"category":"authentication","outcome":"success","type":"access"},"user":{"email":"[email protected]","roles":["Manager", "Security", "Engineering"]}}
2+
{"event":{"category":"authentication","outcome":"success","type":"deletion"},"user":{"email":"[email protected]","roles":["Manager", "Security", "Engineering"]}}

‎examples/terraform/aws/dynamodb/telephone/post_deploy.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
sleep 5
2+
AWS_DEFAULT_REGION=$AWS_REGION python3 ../build/scripts/aws/kinesis/put_records.py substation_edr terraform/aws/dynamodb/telephone/edr_data.jsonl --print-response
3+
AWS_DEFAULT_REGION=$AWS_REGION python3 ../build/scripts/aws/kinesis/put_records.py substation_idp terraform/aws/dynamodb/telephone/idp_data.jsonl --print-response
4+
AWS_DEFAULT_REGION=$AWS_REGION python3 ../build/scripts/aws/kinesis/put_records.py substation_md terraform/aws/dynamodb/telephone/dvc_mgmt_data.jsonl --print-response

0 commit comments

Comments
 (0)