Skip to content

Commit

Permalink
Merge pull request #309 from IFCA-Advanced-Computing/feature-p-values…
Browse files Browse the repository at this point in the history 
…-callback

Add methods to compute p-values in PermutationTestDistanceBased callback
  • Loading branch information
@jaime-cespedes-sisniega
jaime-cespedes-sisniega committed Feb 18, 2024
2 parents df4ea5d + c118df3 commit ecf4317
Show file tree
Hide file tree
Showing 8 changed files with 491 additions and 156 deletions.
134 changes: 67 additions & 67 deletions docs/source/examples/data_drift/MMD_advance.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:58.157477481Z",
"start_time": "2024-02-16T16:12:53.942261427Z"
"end_time": "2024-02-17T19:17:48.453674110Z",
"start_time": "2024-02-17T19:17:44.731749562Z"
}
}
},
Expand All @@ -42,8 +42,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:58.177626709Z",
"start_time": "2024-02-16T16:12:58.156820964Z"
"end_time": "2024-02-17T19:17:48.456838140Z",
"start_time": "2024-02-17T19:17:48.443627264Z"
}
},
"execution_count": 2
Expand Down Expand Up @@ -90,8 +90,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:58.231182651Z",
"start_time": "2024-02-16T16:12:58.166843644Z"
"end_time": "2024-02-17T19:17:48.508728679Z",
"start_time": "2024-02-17T19:17:48.456036506Z"
}
},
"execution_count": 3
Expand All @@ -117,8 +117,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:58.233172037Z",
"start_time": "2024-02-16T16:12:58.211752497Z"
"end_time": "2024-02-17T19:17:48.527449687Z",
"start_time": "2024-02-17T19:17:48.507163830Z"
}
}
},
Expand Down Expand Up @@ -244,8 +244,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:58.325761642Z",
"start_time": "2024-02-16T16:12:58.220690716Z"
"end_time": "2024-02-17T19:17:48.618274285Z",
"start_time": "2024-02-17T19:17:48.523267778Z"
}
}
},
Expand Down Expand Up @@ -292,8 +292,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:59.014362881Z",
"start_time": "2024-02-16T16:12:58.330106479Z"
"end_time": "2024-02-17T19:17:49.664911615Z",
"start_time": "2024-02-17T19:17:48.621951276Z"
}
}
},
Expand Down Expand Up @@ -343,8 +343,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:12:59.025376511Z",
"start_time": "2024-02-16T16:12:59.020884199Z"
"end_time": "2024-02-17T19:17:49.678392697Z",
"start_time": "2024-02-17T19:17:49.671817507Z"
}
},
"execution_count": 7
Expand Down Expand Up @@ -480,8 +480,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:17:46.577275393Z",
"start_time": "2024-02-16T16:12:59.030173823Z"
"end_time": "2024-02-17T19:22:44.701253430Z",
"start_time": "2024-02-17T19:17:49.685187012Z"
}
},
"execution_count": 8
Expand Down Expand Up @@ -545,8 +545,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:17:46.595250026Z",
"start_time": "2024-02-16T16:17:46.577060977Z"
"end_time": "2024-02-17T19:22:44.760266237Z",
"start_time": "2024-02-17T19:22:44.746259927Z"
}
}
},
Expand Down Expand Up @@ -593,8 +593,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:17:46.659949145Z",
"start_time": "2024-02-16T16:17:46.593867783Z"
"end_time": "2024-02-17T19:22:44.785296531Z",
"start_time": "2024-02-17T19:22:44.747126194Z"
}
}
},
Expand Down Expand Up @@ -622,8 +622,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:17:46.662754862Z",
"start_time": "2024-02-16T16:17:46.602440160Z"
"end_time": "2024-02-17T19:22:44.787800781Z",
"start_time": "2024-02-17T19:22:44.747623265Z"
}
}
},
Expand Down Expand Up @@ -664,8 +664,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:17:46.678825618Z",
"start_time": "2024-02-16T16:17:46.610583289Z"
"end_time": "2024-02-17T19:22:44.802033948Z",
"start_time": "2024-02-17T19:22:44.748160859Z"
}
},
"execution_count": 12
Expand Down Expand Up @@ -825,8 +825,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:32.434187179Z",
"start_time": "2024-02-16T16:17:46.624947418Z"
"end_time": "2024-02-17T19:24:33.579552459Z",
"start_time": "2024-02-17T19:22:44.748570864Z"
}
}
},
Expand Down Expand Up @@ -879,8 +879,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:33.007540003Z",
"start_time": "2024-02-16T16:19:32.439104213Z"
"end_time": "2024-02-17T19:24:34.275867733Z",
"start_time": "2024-02-17T19:24:33.585003875Z"
}
},
"execution_count": 14
Expand Down Expand Up @@ -923,8 +923,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:33.011524066Z",
"start_time": "2024-02-16T16:19:33.006432926Z"
"end_time": "2024-02-17T19:24:34.282442755Z",
"start_time": "2024-02-17T19:24:34.278278230Z"
}
}
},
Expand Down Expand Up @@ -973,8 +973,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:33.104789676Z",
"start_time": "2024-02-16T16:19:33.009110935Z"
"end_time": "2024-02-17T19:24:34.300226242Z",
"start_time": "2024-02-17T19:24:34.282242198Z"
}
}
},
Expand Down Expand Up @@ -1032,8 +1032,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:33.466987384Z",
"start_time": "2024-02-16T16:19:33.050152210Z"
"end_time": "2024-02-17T19:24:34.779159458Z",
"start_time": "2024-02-17T19:24:34.297080808Z"
}
}
},
Expand Down Expand Up @@ -1072,8 +1072,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:33.486259158Z",
"start_time": "2024-02-16T16:19:33.472288854Z"
"end_time": "2024-02-17T19:24:34.795749904Z",
"start_time": "2024-02-17T19:24:34.779515645Z"
}
},
"execution_count": 18
Expand Down Expand Up @@ -1104,8 +1104,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:19:33.623751778Z",
"start_time": "2024-02-16T16:19:33.480473726Z"
"end_time": "2024-02-17T19:24:34.855686275Z",
"start_time": "2024-02-17T19:24:34.785007832Z"
}
},
"execution_count": 19
Expand Down Expand Up @@ -1144,8 +1144,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:11.126831863Z",
"start_time": "2024-02-16T16:19:33.490875845Z"
"end_time": "2024-02-17T19:25:12.637290740Z",
"start_time": "2024-02-17T19:24:34.793373179Z"
}
},
"execution_count": 20
Expand Down Expand Up @@ -1188,7 +1188,7 @@
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "a4e345ef8f6e4652a565d59fed3886d7"
"model_id": "7fededc9957d450d9df83a331e6c4482"
}
},
"metadata": {},
Expand All @@ -1200,7 +1200,7 @@
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "5f150463f92c4f05b49eeea4a3b69448"
"model_id": "d838eb10e39442a59dbdd471a4df1e66"
}
},
"metadata": {},
Expand All @@ -1212,7 +1212,7 @@
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "21554996ba2642b8a197637c07969c08"
"model_id": "f185a3e5f92f4dd893371efede2b0861"
}
},
"metadata": {},
Expand All @@ -1224,7 +1224,7 @@
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "342a329a4ec8482681191407865d0795"
"model_id": "71b3e47714b342ccb7c34992e707490d"
}
},
"metadata": {},
Expand All @@ -1242,8 +1242,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:14.682462555Z",
"start_time": "2024-02-16T16:20:11.130324408Z"
"end_time": "2024-02-17T19:25:16.349855710Z",
"start_time": "2024-02-17T19:25:12.640768365Z"
}
},
"execution_count": 21
Expand Down Expand Up @@ -1273,8 +1273,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:14.735987652Z",
"start_time": "2024-02-16T16:20:14.682151132Z"
"end_time": "2024-02-17T19:25:16.399374408Z",
"start_time": "2024-02-17T19:25:16.349624925Z"
}
}
},
Expand Down Expand Up @@ -1342,8 +1342,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:15.963870971Z",
"start_time": "2024-02-16T16:20:14.729968761Z"
"end_time": "2024-02-17T19:25:17.774299178Z",
"start_time": "2024-02-17T19:25:16.397465080Z"
}
},
"execution_count": 23
Expand Down Expand Up @@ -1376,8 +1376,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:15.965661922Z",
"start_time": "2024-02-16T16:20:15.963526707Z"
"end_time": "2024-02-17T19:25:17.776333552Z",
"start_time": "2024-02-17T19:25:17.774063310Z"
}
}
},
Expand Down Expand Up @@ -1417,8 +1417,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:15.978381396Z",
"start_time": "2024-02-16T16:20:15.963740999Z"
"end_time": "2024-02-17T19:25:17.799793047Z",
"start_time": "2024-02-17T19:25:17.774429027Z"
}
},
"execution_count": 25
Expand All @@ -1440,8 +1440,8 @@
" num_permutations=num_permutations,\n",
" random_state=seed,\n",
" num_jobs=-1,\n",
" method=\"exact\",\n",
" name=\"permutation_test\",\n",
" conservative=True,\n",
" verbose=False,\n",
" ),\n",
" ],\n",
Expand All @@ -1451,8 +1451,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:16.181632069Z",
"start_time": "2024-02-16T16:20:15.978662597Z"
"end_time": "2024-02-17T19:25:18.003211081Z",
"start_time": "2024-02-17T19:25:17.792743610Z"
}
}
},
Expand Down Expand Up @@ -1505,8 +1505,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:20:16.402059920Z",
"start_time": "2024-02-16T16:20:16.022002212Z"
"end_time": "2024-02-17T19:25:18.284100029Z",
"start_time": "2024-02-17T19:25:17.833430365Z"
}
},
"execution_count": 27
Expand All @@ -1529,11 +1529,11 @@
"output_type": "stream",
"text": [
"Original (unmodified):\n",
"\tMMD statistic=-0.0003, p-value=0.7219, drift=False\n",
"\tMMD statistic=-0.0003, p-value=0.72185513, drift=False\n",
"Gaussian Blur:\n",
"\tMMD statistic=0.0083, p-value=0.0002, drift=True\n",
"\tMMD statistic=0.0083, p-value=0.00019946, drift=True\n",
"Elastic Transform:\n",
"\tMMD statistic=0.0075, p-value=0.0002, drift=True\n"
"\tMMD statistic=0.0075, p-value=0.00019946, drift=True\n"
]
}
],
Expand All @@ -1555,13 +1555,13 @@
" mmd, callbacks_logs = detector.compare(X=sample)\n",
" permutation_test_logs[type_] = copy.copy(callbacks_logs[\"permutation_test\"])\n",
" mmd, p_value = mmd.distance, callbacks_logs[\"permutation_test\"][\"p_value\"]\n",
" print(f\"{type_}:\\n\\tMMD statistic={round(mmd, 4)}, p-value={round(p_value, 4)}, drift={p_value <= alpha}\")"
" print(f\"{type_}:\\n\\tMMD statistic={round(mmd, 4)}, p-value={round(p_value, 8)}, drift={p_value <= alpha}\")"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:22:17.370932094Z",
"start_time": "2024-02-16T16:20:16.406203620Z"
"end_time": "2024-02-17T19:27:38.781406175Z",
"start_time": "2024-02-17T19:25:18.291176981Z"
}
}
},
Expand Down Expand Up @@ -1648,8 +1648,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-02-16T16:22:19.458873310Z",
"start_time": "2024-02-16T16:22:17.379199703Z"
"end_time": "2024-02-17T19:27:40.819480169Z",
"start_time": "2024-02-17T19:27:38.800648045Z"
}
}
},
Expand Down
Loading

0 comments on commit ecf4317

Please sign in to comment.