-
Notifications
You must be signed in to change notification settings - Fork 9
/
presentation.html
498 lines (438 loc) · 28 KB
/
presentation.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>AB Statistics Presentation</title>
<link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
<link rel="stylesheet" href="reveal/css/reveal.css">
<link rel="stylesheet" href="reveal/css/theme/white.css">
<link rel="stylesheet" href="ab-stats.css">
<script>
var link = document.createElement( 'link' );
link.rel = 'stylesheet';
link.type = 'text/css';
link.href = window.location.search.match( /print-pdf/gi ) ? 'reveal/css/print/pdf.css' : 'reveal/css/print/paper.css';
document.getElementsByTagName( 'head' )[0].appendChild( link );
</script>
</head>
<body>
<div class="reveal">
<div class="slides">
<section id="title">
<h1>Design <span class="title-divider">of</span> experiments</h1>
<p>Statistical foundations for causal inference</p>
<br />
<p>
<small>
<strong>Lukas Vermeer</strong> is <strong>Director</strong> of <strong>Experimentation</strong> at <strong>Booking.com</strong>
</small>
</p>
<p>
<small>
<a href="http://twitter.com/lukasvermeer" target="_blank">@lukasvermeer</a> — <a href="http://www.lukasvermeer.nl/ab-stats" target="_blank">lukasvermeer.nl/ab-stats</a>
</small>
</p>
</section>
<section id="causal-inference">
<span class="text-muted">Making good decisions requires</span>
<h2>causal inference</h2>
<p>
<span class="fragment">Umbrellas often appear just before it pours,</span>
<span class="fragment">but banning them will not stop the rain;</span>
<span class="fragment">it will just make everyone <mark>more wet</mark></span>
</p>
<aside class="notes">
- We want gather evidence for a causal link between a change and behaviour
- “By switching to B, we will cause more people to click ads or buy our product!”
- Correlation is not sufficient for supporting decision making
- Direction of effect is as important as showing there is a relation at all
</aside>
</section>
<section id="app-rubin-full">
<h3>What we want to know <small>(Rubin Causal Model)</small></h3>
<rubin-model display_ate></rubin-model>
</section>
<section id="fundamental-problem">
<span class="text-muted">The fundamental problem of</span>
<h2>causal inference</h2>
<ol>
<li class="fragment">We cannot sample the entire <mark>population</mark></li>
<li class="fragment">We cannot expose units to both treatments <mark>exclusively</mark></li>
<li class="fragment">We cannot directly observe <mark>underlying</mark> probabilities</li>
</ol>
<aside class="notes">
- Other potential, but unquantifiable, sources of uncertainty also exist
- The population may change
- The treatment may change
- The effect may change
- Stability of these other factors is (often implicitly) assumed
</aside>
</section>
<section id="app-rubin-fpci">
<h3>What we can measure <small>(Rubin Causal Model)</small></h3>
<rubin-model sample_name="Sample" randomize reveal_enabled></rubin-model>
<span class="fragment" data-method="rerandomize"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="two-questions">
<span class="text-muted">We will try to answer two</span>
<h2>key questions</h2>
<ol>
<li class="fragment"><mark>Is there</mark> any causal effect?</li>
<li class="fragment">What is the <mark>size</mark> of the causal effect?</li>
</ol>
<p class="fragment text-muted">(If the size is non-zero, there is an effect.)</p>
<aside class="notes">
- Sometimes we will get “unlucky” in our randomization
- We might randomly put all the men in one group and all the women in the other
- We can estimate the likelihood of getting a particular result by chance using a test statistic
- Every randomization will likely give us a slightly different result
- Every time the experiment is repeated with new randomization, the averages will be different
- We can quantify an expected range of results using a confidence interval
</aside>
</section>
<section id="app-rubin-ate">
<h3>What randomization gives us <small>(Rubin Causal Model)</small></h3>
<rubin-model sample_name="Sample" randomize display_ate reveal_enabled></rubin-model>
<span class="fragment" data-method="rerandomize"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-expectation">
<h3>Repeating the same experiment <small>(Expectation)</small></h3>
<simulation v-bind:effect="0.1" hide_significance display_true_effect display_observed_effect reveal_enabled></simulation>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="expectation">
<span class="text-muted">Our answers will be correct</span>
<h2>in expectation</h2>
<p>
<span class="fragment">If we show A and B to random samples of the population,</span>
<span class="fragment">then on average the fraction of yes in both groups will be equal to the true underlying means,</span>
<span class="fragment">and on average across replications of the experiment the difference between the means will be equal to the <mark>average treatment effect</mark></span>
</p>
<aside class="notes">
- In expectation, the average of a random sample will approximate true mean of the population
- In expectation, the fraction of yes will approximate underlying probability
- We do not need to observe the probabilities directly if we have the outcomes for individuals
- In expectation, the difference between averages will give us the ATE
- We do not need to measure the treatment effects of each individual participant
</aside>
</section>
<section id="app-one-trial">
<h3>Up, down or neither?<small>(I don't know either)</small></h3>
<simulation v-bind:effect="Math.random() > 0.5 ? 0 : Math.random() > 0.5 ? 0.1 : -0.1" hide_significance hide_guide display_observed_effect display_last_effect reveal_enabled></simulation>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="randomization">
<span class="text-muted">Randomization ensures only three things can</span>
<h2>explain a difference</h2>
<ol>
<li class="fragment"><mark>Causation</mark> resulted in people behaving differently when treatment was applied</li>
<li class="fragment"><mark>Pure chance</mark> resulted in a difference between the two groups unrelated to the treatment</li>
<li class="fragment"><mark>Mistakes</mark> resulted in an unintended difference in results unrelated to the treatment</li>
</ol>
</section>
<section id="loaded-die">
<span class="text-muted">Is this die</span>
<h2>Fair?</h2>
<p class="dice"><span class="fragment six">⚅</span><span class="fragment six">⚅</span><span class="fragment six">⚅</span></p>
<p>
<span class="fragment">Not fair; I cheated.</span>
</p>
</section>
<section id="decision-rule">
<span class="text-muted">How did you decide you were confident</span>
<h2>that I was cheating?</h2>
<p>
<span class="fragment">You did not need to know what to expect from a loaded die.</span>
<span class="fragment">Instead, you simply <mark>rejected the idea that it was fair</mark>.</span>
</p>
</section>
<section id="null-hypothesis">
<span class="text-muted">We want to reject the</span>
<h2>null hypothesis</h2>
<p>
<span class="fragment">The <mark>null hypothesis</mark> assumes there is no treatment effect for any unit; any difference we observe is simply due to chance</span>
</p>
<p>
<span class="fragment">If we could reasonably rule out mistakes and chance, we might reject the null and consider this to be evidence for an <mark>alternative</mark></span>
</p>
</section>
<section id="p-value">
<span class="text-muted">We compute a</span>
<h2>p-value</h2>
<p>
<span class="fragment">Assuming there is no effect, the p-value is the probability of seeing a particular result or more extreme by chance.</span>
</p>
<p>
<span class="fragment"><strong>How likely is this result <mark>assuming the null is true</mark>?</strong></span>
</p>
<p>
<span class="fragment text-muted">(The probability of rolling three sixes on a fair die is 0.00462962.)</span>
</p>
</section>
<section id="app-rubin-p-value">
<h3>Randomization inference <small>(Rubin Causal Model)</small></h3>
<rubin-model sample_name="Sample" randomize display_ate reveal_enabled></rubin-model>
<span class="fragment" data-method="randomization_inference"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-non-null-p-values">
<span class="text-muted">When the null is <mark>false</mark> p-values</span>
<h3>tend to be small</h3>
<simulation v-bind:effect="0.05" hide_guide hide_significance graph_p_values reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-null-p-values">
<span class="text-muted">When the null is <mark>true</mark> p-values will be</span>
<h3>uniformly distributed</h3>
<simulation v-bind:effect="0.0" hide_guide hide_significance graph_p_values reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="significance-threshold">
<span class="text-muted">We need to pick a</span>
<h2>threshold</h2>
<p>
<span class="fragment">One swallow does not a summer make, nor one fine day,</span>
<span class="fragment">but how many swallows do we count before we pack away our umbrellas?</span>
</p>
<p>
<span class="fragment"><strong>Scientific standard for significance:</strong> <mark>p < 0.05</mark></span>
</p>
<aside class="notes">
- Sometimes we will get “unlucky” in our randomization
- We will have to decide at which point we will reject the idea that our result could be chance
- At this point, we will accept the alternative hypothesis and argue there is a causal effect
</aside>
</section>
<section id="app-p-values-significance">
<span class="text-muted">Results under our threshold are called</span>
<h3>statistically significant</h3>
<simulation v-bind:effect="0.05" hide_guide graph_p_values reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<!--
<section id="misinterpreted-p-value">
<span class="text-muted">p-values are often<small>[2]</small></span>
<h2>misinterpreted</h2>
<span class="fragment">Some examples of <mark>incorrect</mark> interpretations</span>
<ol>
<li class="fragment">p = .05 means the null hypothesis has only a 5% chance of being true</li>
<li class="fragment">A non-significant difference (e.g. p > .05) means there is no difference between groups</li>
<li class="fragment">p = .05 means that we have observed data that would occur only 5% of the time under the null</li>
<li class="fragment">p = .05 means that if you reject the null, the probability of a type I error (false positive) is 5%</li>
</ol>
<p><small>[2] Goodman, Steve 2008. “A dirty dozen: twelve P-value misconceptions.” Seminars in Hematology, 45 (2008), pp. 135-140.</small></p>
<aside class="notes">
- p-value gives us Prob(X >= x | H0), whereas what we want is Prob(H0 | X = x)
- Here are some incorrect statements from Steve Goodman’s A Dirty Dozen[2]
</aside>
</section>
-->
<section id="errors">
<span class="text-muted">Two types of</span>
<h2>errors</h2>
<ol>
<li class="fragment"><mark>Type-I</mark> is the incorrect rejection of a true null hypothesis; <span class="fragment">we cried foul when there was none</span></li>
<li class="fragment"><mark>Type-II</mark> is the failure to reject a false null hypothesis; <span class="fragment">we failed to detect a real effect</span></li>
</ol>
<aside class="notes">
- The result might simply not be that unlikely when assuming the null is true
</aside>
</section>
<section id="app-type-i">
<h3>Repeating the same experiment <small>(No effect)</small></h3>
<simulation v-bind:effect="0.0" display_true_effect display_type_i reveal_enabled></simulation>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-type-ii">
<h3>Repeating the same experiment <small>(Small effect)</small></h3>
<simulation v-bind:effect="0.05" display_true_effect display_type_ii reveal_enabled></simulation>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="power">
<span class="text-muted">The importance of</span>
<h2>Statistical power</h2>
<p>
<span class="fragment">Statistical power is the probability that the test <mark>correctly rejects</mark> the null hypothesis when the alternative hypothesis is true</span>
</p>
<p>
<span class="fragment"><strong>Two main things affect statistical power:</strong></span>
<ul>
<li class="fragment">Sample size <span class="text-muted">(more is better)</span></li>
<li class="fragment">Effect size <span class="text-muted">(more is better)</span></li>
</ul>
</p>
<aside class="notes">
- Effect size is unknown but assumed fixed
- Sample size may be increased is low power is expected
</aside>
</section>
<section id="app-type-ii-2">
<h3>Repeating the same experiment <small>(More power)</small></h3>
<simulation v-bind:effect="0.05" v-bind:n="2000" display_type_ii display_power reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-type-ii-3">
<h3>Repeating the same experiment <small>(MOAR POWER!!11)</small></h3>
<simulation v-bind:effect="0.05" v-bind:n="4000" display_type_ii display_power reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-one-trial-2">
<h3>Up, down or neither?<small>(I don't know either)</small></h3>
<simulation v-bind:effect="Math.random() > 0.5 ? 0 : Math.random() > 0.5 ? 0.1 : -0.1" hide_guide display_observed_effect display_last_effect reveal_enabled></simulation>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="step"><small class="text-very-muted"><i class="glyphicon glyphicon-step-forward"></i></small></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="protocol">
<span class="text-muted">The importance of sticking to</span>
<h2>Protocol</h2>
<p>
<span class="fragment">The methods described assume strict adherence to protocol;</span>
<span class="fragment">violations of protocol such as <mark>peeking</mark> and <mark>multiple testing</mark> increase the type-I error rate</span>
</p>
</section>
<section id="app-dice">
<span class="text-muted">Can telekenisis influence</span>
<h2>Three dice?</h2>
<span class="fragment"><dice reveal_enabled></dice></span>
<span class="fragment" data-method="rerandomize"></span>
<span class="fragment" data-method="toggle_repeat" trigger-hidden></span>
<p>
<span class="fragment">Fair die; I still cheated.</span>
</p>
<p>
<span class="fragment text-muted"><small>(The probability of rolling three sixes on a fair die <mark>if you keep trying</mark> is 1.)</small></span>
</p>
</section>
<section id="app-multiple-testing">
<h3>Repeating the same experiment <small>(Peeking twice)</small></h3>
<simulation v-bind:effect="0.00" v-bind:peek="2" display_true_effect display_type_i reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="app-multiple-testing-2">
<h3>Repeating the same experiment <small>(Peeking 100x)</small></h3>
<simulation v-bind:effect="0.00" v-bind:peek="100" display_true_effect display_type_i reveal_enabled></simulation>
<span class="fragment" data-method="toggle_repeat" trigger-hidden><small class="text-very-muted"><i class="glyphicon glyphicon-forward"></i></small></span>
</section>
<section id="adaptive-methods">
<span class="text-muted">Reasons for violating</span>
<h2>Protocol</h2>
<p>
<span class="fragment">More flexible protocols may be desirable</span>
<ul>
<li class="fragment"><mark>early stopping rules</mark> to mitigate damage</li>
<li class="fragment"><mark>early shipping</mark> to minimize opportunity cost</li>
<li class="fragment"><mark>multiple variants</mark> to test several alternatives</li>
<li class="fragment"><mark>multiple metrics</mark> to guard business KPIs</li>
</ul>
</p>
<div class="fragment">
<p>All these are possible<small>[4]</small>, but require protocol adjustments</p>
<p><small>[4] Alex Deng, Tianxi Li, Yu Guo 2014 “Statistical Inference in Two-Stage Online Controlled Experiments with Treatment Selection and Validation” WWW '14. 609–618.</small></p>
</div>
</section>
<section id="end">
<h1>Design <span class="title-divider">of</span> experiments</h1>
<p>Statistical foundations for causal inference</p>
<br />
<p>
<small>
<strong>Lukas Vermeer</strong> is <strong>Director</strong> of <strong>Experimentation</strong> at <strong>Booking.com</strong>
</small>
</p>
<p>
<small>
<a href="http://twitter.com/lukasvermeer" target="_blank">@lukasvermeer</a> — <a href="http://www.lukasvermeer.nl/ab-stats" target="_blank">lukasvermeer.nl/ab-stats</a>
</small>
</p>
</section>
<section id="references">
<h2>References</h2>
<ol>
<li><small>Rubin, Donald B. 1974. “Estimating Causal Effects of Treatments in Randomized and Nonrandomized Studies.” Journal of Educational Psychology 66 (5): 688–701. (<a href="http://dx.doi.org/10.1037/h0037350" target="_blank">link</a>)</small></li>
<li><small>Goodman, Steve 2008. “A dirty dozen: twelve P-value misconceptions.” Seminars in Hematology, 45 (2008), pp. 135-140. (<a href="https://www.researchgate.net/publication/5272766_A_Dirty_Dozen_Twelve_P-Value_Misconceptions" target="_blank">link</a>)</small></li>
<li><small>Kohavi, R., Longbotham, R., Sommerfield, D. et al. 2009 “Controlled experiments on the web: survey and practical guide” Data Min Knowl Disc (2009) 18: 140. (<a href="http://bit.ly/expSurvey" target="_blank">link</a>)</small></li>
<li><small>Alex Deng, Tianxi Li, Yu Guo 2014 “Statistical Inference in Two-Stage Online Controlled Experiments with Treatment Selection and Validation” WWW '14. 609–618.(<a href="http://www.exp-platform.com/Documents/p609-deng.pdf" target="_blank">link</a>)</small></li>
</ol>
</section>
</div>
</div>
<script type="text/javascript" src="d3/d3.js"></script>
<script type="text/javascript" src="jstat/jstat.min.js"></script>
<script type="text/javascript" src="vue/vue.min.js"></script>
<script type="text/javascript" src="vue-components.js"></script>
<script type="text/javascript" src="reveal/lib/js/head.min.js"></script>
<script type="text/javascript" src="reveal/js/reveal.js"></script>
<script type="text/javascript">
Reveal.initialize({
controls: false,
progress: true,
history: true,
center: true,
width: 1140,
height: 960,
margin: 0.05,
slideNumber: 'c/t',
transition: 'slide',
dependencies: [
{ src: 'reveal/plugin/notes/notes.js', async: true }
]
});
const EventBus = new Vue();
new Vue({ el: '#app-rubin-full' });
new Vue({ el: '#app-rubin-fpci' });
new Vue({ el: '#app-rubin-ate' });
new Vue({ el: '#app-rubin-p-value' });
new Vue({ el: '#app-expectation' });
new Vue({ el: '#app-one-trial' });
new Vue({ el: '#app-one-trial-2' });
new Vue({ el: '#app-non-null-p-values' });
new Vue({ el: '#app-null-p-values' });
new Vue({ el: '#app-p-values-significance' });
new Vue({ el: '#app-type-i' });
new Vue({ el: '#app-type-ii' });
new Vue({ el: '#app-type-ii-2' });
new Vue({ el: '#app-type-ii-3' });
new Vue({ el: '#app-multiple-testing' });
new Vue({ el: '#app-multiple-testing-2' });
new Vue({ el: '#app-dice' });
Reveal.addEventListener( 'fragmentshown', function( event ) {
if (event.fragment.hasAttribute('data-method')) {
EventBus.$emit('fragmentMethod', { id: event.fragment.parentNode.id, method: event.fragment.getAttribute('data-method')});
}
} );
Reveal.addEventListener( 'fragmenthidden', function( event ) {
if (event.fragment.hasAttribute('data-method') && event.fragment.hasAttribute('trigger-hidden')) {
EventBus.$emit('fragmentMethod', { id: event.fragment.parentNode.id, method: event.fragment.getAttribute('data-method')});
}
} );
if (window.location.search.match( /print-pdf/gi )) {
EventBus.$emit('fragmentMethod', { id: 'app-expectation', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-type-i', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-type-ii', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-type-ii-2', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-type-ii-3', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-multiple-testing', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-multiple-testing-2', method: 'toggle_repeat'});
EventBus.$emit('fragmentMethod', { id: 'app-dice', method: 'toggle_repeat'});
}
</script>
</body>
</html>