Skip to content

Commit 1c1f77e

Browse files
rustworthyrustworthy
authored
Seed DynamoDB locally as well (#197)
1 parent 3b76c55 commit 1c1f77e

File tree

4 files changed

+164
-83
lines changed

4 files changed

+164
-83
lines changed

‎CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ auto-update. If you modify files under `server/`, you'll have to re-run
2727
Note that when run this way, to aid in development, the server will
2828
auto-populate an event with a set of questions from a past live Q&A
2929
session I ran at
30-
<http://localhost:5173/#/event/00000000-0000-0000-0000-000000000000/secret>.
30+
<http://localhost:5173/event/00000000000000000000000000/secret>.
3131
It will also auto-generate user votes over time for the questions there.
3232

3333
If you're curious about the technologies used in the server and client,

‎server/src/ask.rs

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use super::{Backend, Local};
2-
use crate::to_dynamo_timestamp;
2+
use crate::{to_dynamo_timestamp, QUESTIONS_TTL};
33
use aws_sdk_dynamodb::{
44
error::SdkError,
55
operation::put_item::{PutItemError, PutItemOutput},
@@ -9,17 +9,12 @@ use axum::extract::{Path, State};
99
use axum::response::Json;
1010
use http::StatusCode;
1111
use serde::Deserialize;
12-
use std::{
13-
collections::HashMap,
14-
time::{Duration, SystemTime},
15-
};
12+
use std::{collections::HashMap, time::SystemTime};
1613
use ulid::Ulid;
1714

1815
#[allow(unused_imports)]
1916
use tracing::{debug, error, info, trace, warn};
2017

21-
const QUESTIONS_EXPIRE_AFTER_DAYS: u64 = 30;
22-
2318
impl Backend {
2419
pub(super) async fn ask(
2520
&self,
@@ -35,10 +30,7 @@ impl Backend {
3530
("when", to_dynamo_timestamp(SystemTime::now())),
3631
(
3732
"expire",
38-
to_dynamo_timestamp(
39-
SystemTime::now()
40-
+ Duration::from_secs(QUESTIONS_EXPIRE_AFTER_DAYS * 24 * 60 * 60),
41-
),
33+
to_dynamo_timestamp(SystemTime::now() + QUESTIONS_TTL),
4234
),
4335
("hidden", AttributeValue::Bool(false)),
4436
];

‎server/src/main.rs

Lines changed: 157 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use axum::Router;
66
use http::StatusCode;
77
use http_body_util::BodyExt;
88
use lambda_http::Error;
9-
use std::time::SystemTime;
9+
use std::time::{Duration, SystemTime};
1010
use std::{
1111
collections::HashMap,
1212
future::Future,
@@ -19,6 +19,12 @@ use tower_service::Service;
1919
use tracing_subscriber::EnvFilter;
2020
use ulid::Ulid;
2121

22+
const QUESTIONS_EXPIRE_AFTER_DAYS: u64 = 30;
23+
const QUESTIONS_TTL: Duration = Duration::from_secs(QUESTIONS_EXPIRE_AFTER_DAYS * 24 * 60 * 60);
24+
25+
const EVENTS_EXPIRE_AFTER_DAYS: u64 = 60;
26+
const EVENTS_TTL: Duration = Duration::from_secs(EVENTS_EXPIRE_AFTER_DAYS * 24 * 60 * 60);
27+
2228
#[allow(unused_imports)]
2329
use tracing::{debug, error, info, trace, warn};
2430

@@ -33,7 +39,6 @@ enum Backend {
3339
}
3440

3541
impl Backend {
36-
#[cfg(test)]
3742
async fn local() -> Self {
3843
Backend::Local(Arc::new(Mutex::new(Local::default())))
3944
}
@@ -178,83 +183,172 @@ fn mint_service_error<E>(e: E) -> SdkError<E> {
178183
)
179184
}
180185

186+
/// Seed the database.
187+
///
188+
/// This will register a test event (with id `00000000000000000000000000`) and
189+
/// a number of questions for it in the database, whether it's an in-memory [`Local`]
190+
/// database or a local instance of DynamoDB. Note that in the latter case
191+
/// we are checking if the test event is already there, and - if so - we are _not_ seeding
192+
/// the questions. This is to avoid creating duplicated questions when re-running the app.
193+
/// And this is not an issue of course when running against our in-memory [`Local`] database.
194+
///
195+
/// The returned vector contains IDs of the questions related to the test event.
196+
#[cfg(debug_assertions)]
197+
async fn seed(backend: &mut Backend) -> Vec<Ulid> {
198+
#[derive(serde::Deserialize)]
199+
struct LiveAskQuestion {
200+
likes: usize,
201+
text: String,
202+
hidden: bool,
203+
answered: bool,
204+
#[serde(rename = "createTimeUnix")]
205+
created: usize,
206+
}
207+
208+
let seed: Vec<LiveAskQuestion> = serde_json::from_str(SEED).unwrap();
209+
let seed_e = Ulid::from_string("00000000000000000000000000").unwrap();
210+
let seed_e_secret = "secret";
211+
212+
info!("going to seed test event");
213+
match backend.event(&seed_e).await.unwrap() {
214+
output if output.item().is_some() => {
215+
warn!("test event is already there, skipping seeding questions");
216+
}
217+
_ => {
218+
backend.new(&seed_e, seed_e_secret).await.unwrap();
219+
info!("successfully registered test event, going to seed questions now");
220+
// first create questions ...
221+
let mut qs = Vec::new();
222+
for q in seed {
223+
let qid = ulid::Ulid::new();
224+
backend
225+
.ask(
226+
&seed_e,
227+
&qid,
228+
ask::Question {
229+
body: q.text,
230+
asker: None,
231+
},
232+
)
233+
.await
234+
.unwrap();
235+
qs.push((qid, q.created, q.likes, q.hidden, q.answered));
236+
}
237+
// ... then set the vote count + answered/hidden flags
238+
match backend {
239+
Backend::Dynamo(ref mut client) => {
240+
use aws_sdk_dynamodb::types::BatchStatementRequest;
241+
// DynamoDB supports batch operations using PartiQL syntax with `25` as max batch size
242+
// https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchExecuteStatement.html
243+
for chunk in qs.chunks(25) {
244+
let batch_update = chunk
245+
.iter()
246+
.map(|(qid, created, votes, hidden, answered)| {
247+
let builder = BatchStatementRequest::builder();
248+
let builder = if *answered {
249+
builder.statement(
250+
// numerous words are reserved in the DynamoDB engine (e.g. Key, Id, When) and
251+
// should be qouted; we are quoting all of our attrs to avoid possible collisions
252+
r#"UPDATE "questions" SET "answered"=? SET "votes"=? SET "when"=? SET "hidden"=? WHERE "id"=?"#,
253+
)
254+
.parameters(to_dynamo_timestamp(SystemTime::now())) // answered
255+
} else {
256+
builder.statement(
257+
r#"UPDATE "questions" SET "votes"=? SET "when"=? SET "hidden"=? WHERE "id"=?"#,
258+
)
259+
};
260+
builder
261+
.parameters(AttributeValue::N(votes.to_string())) // votes
262+
.parameters(AttributeValue::N(created.to_string())) // when
263+
.parameters(AttributeValue::Bool(*hidden)) // hidden
264+
.parameters(AttributeValue::S(qid.to_string())) // id
265+
.build()
266+
.unwrap()
267+
})
268+
.collect::<Vec<_>>();
269+
client
270+
.batch_execute_statement()
271+
.set_statements(Some(batch_update))
272+
.send()
273+
.await
274+
.expect("batch to have been written ok");
275+
}
276+
}
277+
Backend::Local(ref mut state) => {
278+
let state = Arc::get_mut(state).unwrap();
279+
let state = Mutex::get_mut(state).unwrap();
280+
for (qid, created, votes, hidden, answered) in qs {
281+
let q = state.questions.get_mut(&qid).unwrap();
282+
q.insert("votes", AttributeValue::N(votes.to_string()));
283+
if answered {
284+
q.insert("answered", to_dynamo_timestamp(SystemTime::now()));
285+
}
286+
q.insert("hidden", AttributeValue::Bool(hidden));
287+
q.insert("when", AttributeValue::N(created.to_string()));
288+
}
289+
}
290+
}
291+
info!("successfully registered questions");
292+
}
293+
}
294+
// let's collect ids of the questions related to the test event,
295+
// we can then use them to auto-generate user votes over time
296+
backend
297+
.list(&seed_e, true)
298+
.await
299+
.expect("scenned index ok")
300+
.items()
301+
.iter()
302+
.filter_map(|item| {
303+
let id = item
304+
.get("id")
305+
.expect("id is in projection")
306+
.as_s()
307+
.expect("id is of type string");
308+
ulid::Ulid::from_string(id).ok()
309+
})
310+
.collect()
311+
}
312+
181313
#[tokio::main]
182314
async fn main() -> Result<(), Error> {
183315
tracing_subscriber::fmt()
184316
.with_env_filter(EnvFilter::from_default_env())
317+
// TODO: we may _not_ want `without_time` when deploying
318+
// TODO: on non-Lambda runtimes; this can be addressed as
319+
// TODO: part of https://github.com/jonhoo/wewerewondering/issues/202
185320
.without_time(/* cloudwatch does that */).init();
186321

187322
#[cfg(not(debug_assertions))]
188323
let backend = Backend::dynamo().await;
324+
189325
#[cfg(debug_assertions)]
190-
let backend = if std::env::var_os("USE_DYNAMODB").is_some() {
191-
Backend::dynamo().await
192-
} else {
326+
let backend = {
193327
use rand::prelude::SliceRandom;
194-
use serde::Deserialize;
195-
use std::time::Duration;
196-
197-
#[cfg(debug_assertions)]
198-
#[derive(Deserialize)]
199-
struct LiveAskQuestion {
200-
likes: usize,
201-
text: String,
202-
hidden: bool,
203-
answered: bool,
204-
#[serde(rename = "createTimeUnix")]
205-
created: usize,
206-
}
207328

208-
let mut state = Local::default();
209-
let seed: Vec<LiveAskQuestion> = serde_json::from_str(SEED).unwrap();
210-
let seed_e = "00000000000000000000000000";
211-
let seed_e = Ulid::from_string(seed_e).unwrap();
212-
state.events.insert(seed_e, String::from("secret"));
213-
state.questions_by_eid.insert(seed_e, Vec::new());
214-
let mut state = Backend::Local(Arc::new(Mutex::new(state)));
215-
let mut qs = Vec::new();
216-
for q in seed {
217-
let qid = ulid::Ulid::new();
218-
state
219-
.ask(
220-
&seed_e,
221-
&qid,
222-
ask::Question {
223-
body: q.text,
224-
asker: None,
225-
},
226-
)
227-
.await
228-
.unwrap();
229-
qs.push((qid, q.created, q.likes, q.hidden, q.answered));
230-
}
231-
let mut qids = Vec::new();
232-
{
233-
let Backend::Local(ref mut state): Backend = state else {
234-
unreachable!();
235-
};
236-
let state = Arc::get_mut(state).unwrap();
237-
let state = Mutex::get_mut(state).unwrap();
238-
for (qid, created, votes, hidden, answered) in qs {
239-
let q = state.questions.get_mut(&qid).unwrap();
240-
q.insert("votes", AttributeValue::N(votes.to_string()));
241-
if answered {
242-
q.insert("answered", to_dynamo_timestamp(SystemTime::now()));
243-
}
244-
q.insert("hidden", AttributeValue::Bool(hidden));
245-
q.insert("when", AttributeValue::N(created.to_string()));
246-
qids.push(qid);
247-
}
248-
}
249-
let cheat = state.clone();
329+
let mut backend = if std::env::var_os("USE_DYNAMODB").is_some() {
330+
Backend::dynamo().await
331+
} else {
332+
Backend::local().await
333+
};
334+
335+
// to aid in development, seed the backend with a test event and related
336+
// questions, and auto-generate user votes over time
337+
let qids = seed(&mut backend).await;
338+
let cheat = backend.clone();
250339
tokio::spawn(async move {
340+
let mut interval = tokio::time::interval(Duration::from_secs(1));
341+
interval.tick().await;
251342
loop {
252-
tokio::time::sleep(Duration::from_secs(1)).await;
253-
let qid = qids.choose(&mut rand::thread_rng()).unwrap();
343+
interval.tick().await;
344+
let qid = qids
345+
.choose(&mut rand::thread_rng())
346+
.expect("there _are_ some questions for our test event");
254347
let _ = cheat.vote(qid, vote::UpDown::Up).await;
255348
}
256349
});
257-
state
350+
351+
backend
258352
};
259353

260354
let app = Router::new()

‎server/src/new.rs

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::to_dynamo_timestamp;
1+
use crate::{to_dynamo_timestamp, EVENTS_TTL};
22

33
use super::{Backend, Local};
44
use aws_sdk_dynamodb::{
@@ -11,14 +11,12 @@ use axum::response::Json;
1111
use http::StatusCode;
1212
use rand::distributions::Alphanumeric;
1313
use rand::{thread_rng, Rng};
14-
use std::time::{Duration, SystemTime};
14+
use std::time::SystemTime;
1515
use ulid::Ulid;
1616

1717
#[allow(unused_imports)]
1818
use tracing::{debug, error, info, trace, warn};
1919

20-
const EVENTS_EXPIRE_AFTER_DAYS: u64 = 60;
21-
2220
impl Backend {
2321
#[allow(clippy::wrong_self_convention)]
2422
#[allow(clippy::new_ret_no_self)]
@@ -37,10 +35,7 @@ impl Backend {
3735
.item("when", to_dynamo_timestamp(SystemTime::now()))
3836
.item(
3937
"expire",
40-
to_dynamo_timestamp(
41-
SystemTime::now()
42-
+ Duration::from_secs(EVENTS_EXPIRE_AFTER_DAYS * 24 * 60 * 60),
43-
),
38+
to_dynamo_timestamp(SystemTime::now() + EVENTS_TTL),
4439
)
4540
.send()
4641
.await

0 commit comments

Comments
 (0)