04_calculate_ground_truth_accuracy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1e5d0454-5940-4d6f-a00c-ff6ecad18219",
   "metadata": {},
   "source": [
    "# Simulated PIK statistics\n",
    "\n",
    "Here we inspect the accuracy and characteristics of the PIKs assigned,\n",
    "leveraging our knowledge of ground truth from pseudopeople.\n",
    "\n",
    "It wouldn't be possible to do the ground truth part with the real PVS, but\n",
    "Layne, Wagner, and Rothhaas did something similar by redacting SSN from real records,\n",
    "sending them through PVS without the SSN, and then using the true SSN\n",
    "as ground truth.\n",
    "The health care records they used are probably quite different from a CUF,\n",
    "but they found a **very** good overall PIK accuracy (see cell below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb8ea2c3-d584-49df-9fb8-e0e681d7fddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Query planning is now on by default, but it has some rough edges.\n",
    "# See https://github.com/dask/dask/issues/10995 for general discussion\n",
    "# and https://github.com/dask/dask-expr/issues/1060 for the particular\n",
    "# issue I ran into.\n",
    "try:\n",
    "    import dask\n",
    "\n",
    "    dask.config.set({\"dataframe.query-planning\": False})\n",
    "except ImportError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "257863f1-5fdb-4c13-a63b-ce53f3b4d92e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datetime, os, time\n",
    "\n",
    "from person_linkage_case_study_utils import distributed_compute, utils\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73667681-3b15-4b87-b387-cd02ea09c53b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(datetime.datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a241f483-7c17-4c52-b2ef-2cde79908b2c",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# DO NOT EDIT if this notebook is not called ground_truth_accuracy.ipynb!\n",
    "# This notebook is designed to be run with papermill; this cell is tagged 'parameters'\n",
    "data_to_use = \"small_sample\"\n",
    "case_study_files_dir = \"output/02_generate_case_study_files\"\n",
    "case_study_output_dir = \"output/03_link_datasets\"\n",
    "\n",
    "compute_engine = \"pandas\"\n",
    "# Only matter if using a distributed compute engine\n",
    "compute_engine_num_workers = 3\n",
    "compute_engine_cpus_per_worker = 2\n",
    "compute_engine_threads_per_worker = 1\n",
    "compute_engine_memory_per_worker = \"1GB\"\n",
    "queue = None\n",
    "account = None\n",
    "# NOTE: This is, as Dask requests, a directory local to the compute node.\n",
    "# But IHME's cluster doesn't support this very well -- it can be small-ish,\n",
    "# full of stuff from other users, etc.\n",
    "compute_engine_local_directory = (\n",
    "    f\"/tmp/{os.environ['USER']}_{int(time.time())}_person_linkage_case_study\"\n",
    ")\n",
    "compute_engine_log_directory = f\"{case_study_output_dir}/{data_to_use}/logs\"\n",
    "walltime = None\n",
    "compute_engine_memory_constrained = True\n",
    "scheduler = \"slurm\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5ebdf21",
   "metadata": {},
   "outputs": [],
   "source": [
    "if compute_engine.startswith(\"dask\"):\n",
    "    utils.ensure_empty(compute_engine_local_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78e187bb-cca2-4290-a296-2ccc29cb600f",
   "metadata": {},
   "outputs": [],
   "source": [
    "case_study_output_dir = f\"{case_study_output_dir}/{data_to_use}\"\n",
    "case_study_files_dir = f\"{case_study_files_dir}/{data_to_use}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a179a449-7544-43a3-ad3e-6384beefc593",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ops, pd = distributed_compute.start_compute_engine(\n",
    "    compute_engine,\n",
    "    num_workers=compute_engine_num_workers,\n",
    "    cpus_per_worker=compute_engine_cpus_per_worker,\n",
    "    threads_per_worker=compute_engine_threads_per_worker,\n",
    "    memory_per_worker=compute_engine_memory_per_worker,\n",
    "    worker_walltime=walltime,\n",
    "    local_directory=compute_engine_local_directory,\n",
    "    log_directory=compute_engine_log_directory,\n",
    "    memory_constrained=compute_engine_memory_constrained,\n",
    "    scheduler=scheduler,\n",
    "    queue=queue,\n",
    "    account=account,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22903d9d-a4ff-4913-b558-9112ab7b0b10",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "census_2030_piked = df_ops.read_parquet(\n",
    "    f\"{case_study_output_dir}/census_2030_piked.parquet\"\n",
    ")\n",
    "confirmed_piks_with_ground_truth = df_ops.read_parquet(\n",
    "    f\"{case_study_output_dir}/confirmed_piks.parquet\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7744423-adad-4abe-9a30-927db60190c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "piked_proportion = df_ops.compute(census_2030_piked.pik.notnull().mean())\n",
    "# Compare with 90.28% of input records PIKed in the 2010 CUF,\n",
    "# as reported in Wagner and Layne, Table 2, p. 18\n",
    "print(f\"{piked_proportion:.2%} of the input records were PIKed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3bfac51-e19c-465c-92ce-5743445db8bb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Multiple Census rows assigned the same PIK, indicating the model thinks they are duplicates in Census\n",
    "pik_sizes = df_ops.persist(\n",
    "    df_ops.groupby_agg_small_groups(\n",
    "        census_2030_piked, by=\"pik\", agg_func=lambda x: x.size()\n",
    "    )\n",
    ")\n",
    "df_ops.compute(pik_sizes.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "163cd903-c5c9-41c5-ac35-480e9db1b7a4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Interesting: in pseudopeople, sometimes siblings are assigned the same (common) first name, making them almost identical.\n",
    "# The only giveaway is their age and DOB.\n",
    "# Presumably, this tends not to happen in real life.\n",
    "duplicate_piks = (\n",
    "    pik_sizes.rename(\"pik_size\").reset_index().pipe(lambda df: df[df.pik_size > 1])\n",
    ")\n",
    "\n",
    "df_ops.head(census_2030_piked.merge(duplicate_piks, on=\"pik\").sort_values(\"pik\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7681c6f5-098e-4674-ac13-64af1bccf98d",
   "metadata": {},
   "source": [
    "## Ground truth statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48dee5fb-d362-4c82-ac4a-c0ae5155879f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "census_2030_ground_truth = df_ops.persist(\n",
    "    df_ops.read_parquet(\n",
    "        f\"{case_study_files_dir}/simulated_census_2030_ground_truth.parquet\"\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a23fbbaa-9a30-4c53-ab1a-a88eedcb88f8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# In this version of pseudopeople, there are no actual duplicates in Census,\n",
    "# which means all of the duplicates identified above are wrong.\n",
    "assert len(census_2030_ground_truth) == len(\n",
    "    df_ops.drop_duplicates(census_2030_ground_truth)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1439858-dde0-4199-8b25-f58dccfe59e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "reference_files_ground_truth = df_ops.persist(\n",
    "    df_ops.concat(\n",
    "        [\n",
    "            df_ops.read_parquet(\n",
    "                f\"{case_study_files_dir}/simulated_geobase_reference_file_ground_truth.parquet\"\n",
    "            ).drop(columns=[\"n_unique_simulants\"]),\n",
    "            df_ops.read_parquet(\n",
    "                f\"{case_study_files_dir}/simulated_name_dob_reference_file_ground_truth.parquet\"\n",
    "            ).drop(columns=[\"n_unique_simulants\"]),\n",
    "        ],\n",
    "        ignore_index=True,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77d710c5-4ab3-4e40-8189-ac9561a5db13",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# However, there can be reference file records that correspond to multiple simulants,\n",
    "# due to errors in the reference file construction by SSN\n",
    "n_unique_simulants = df_ops.persist(\n",
    "    df_ops.groupby_agg_small_groups(\n",
    "        reference_files_ground_truth,\n",
    "        by=\"record_id\",\n",
    "        agg_func=lambda x: x.simulant_id.nunique(),\n",
    "    )\n",
    "    .rename(\"n_unique_simulants\")\n",
    "    .reset_index()\n",
    ")\n",
    "df_ops.compute(n_unique_simulants.n_unique_simulants.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b268d6a-8e6a-4e75-8d3a-4ad4df562f68",
   "metadata": {},
   "outputs": [],
   "source": [
    "reference_files_ground_truth = df_ops.persist(\n",
    "    reference_files_ground_truth.merge(\n",
    "        n_unique_simulants,\n",
    "        on=\"record_id\",\n",
    "        how=\"left\",\n",
    "    )\n",
    ")\n",
    "reference_files_ground_truth.head(n=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db065d7c-75b4-4ae2-a576-c4061a5ce98c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_ops.head(\n",
    "    reference_files_ground_truth[\n",
    "        reference_files_ground_truth.n_unique_simulants\n",
    "        == df_ops.compute(reference_files_ground_truth.n_unique_simulants.max())\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a37eb52-1372-4de9-ab99-a3facecaee48",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "census_2030_ground_truth = df_ops.persist(\n",
    "    census_2030_ground_truth.merge(\n",
    "        df_ops.drop_duplicates(reference_files_ground_truth[[\"simulant_id\"]]).assign(\n",
    "            possible_to_pik=1\n",
    "        ),\n",
    "        on=\"simulant_id\",\n",
    "        how=\"left\",\n",
    "    ).assign(possible_to_pik=lambda df: df.possible_to_pik.fillna(0))\n",
    ")\n",
    "possible_to_pik_proportion = df_ops.compute(\n",
    "    census_2030_ground_truth.possible_to_pik.mean()\n",
    ")\n",
    "print(\n",
    "    f\"{(1 - possible_to_pik_proportion):.2%} of the input records are \"\n",
    "    \"impossible to PIK correctly, since they are not in any reference files\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f4bbb22-d2a0-460a-aa4a-3d160f3f6f5c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\n",
    "    f\"Assigned PIKs to {(piked_proportion / possible_to_pik_proportion):.2%} of PIK-able records\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e06b3e4-e881-4740-82e9-96a3e868ce82",
   "metadata": {},
   "outputs": [],
   "source": [
    "reference_file = df_ops.concat(\n",
    "    [\n",
    "        df_ops.read_parquet(\n",
    "            f\"{case_study_files_dir}/simulated_geobase_reference_file.parquet\",\n",
    "        ),\n",
    "        df_ops.read_parquet(\n",
    "            f\"{case_study_files_dir}/simulated_name_dob_reference_file.parquet\",\n",
    "        ),\n",
    "    ],\n",
    "    ignore_index=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94fa41b9-dd7c-4ca7-a77c-36d7c86f1bb1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "reference_file_piks = df_ops.persist(reference_file[[\"record_id\", \"pik\"]])\n",
    "reference_file_piks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3058bb9-4dd9-4e33-a6f1-1e499871038f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assert len(reference_file_piks) == len(\n",
    "    df_ops.drop_duplicates(reference_file_piks[[\"record_id\"]])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8486907-802c-4442-ad1d-afb6ee0ba2f2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pik_simulant_pairs = df_ops.persist(\n",
    "    df_ops.drop_duplicates(\n",
    "        reference_files_ground_truth.merge(reference_file_piks, on=\"record_id\")[\n",
    "            [\"pik\", \"simulant_id\"]\n",
    "        ]\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c79baaa-7918-4746-b6fd-d484b0767c77",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# However, there can be PIKs that correspond to multiple simulants,\n",
    "# due to errors in the reference file construction by SSN\n",
    "n_unique_simulants = df_ops.persist(\n",
    "    df_ops.groupby_agg_small_groups(\n",
    "        pik_simulant_pairs, by=\"pik\", agg_func=lambda x: x.simulant_id.nunique()\n",
    "    )\n",
    "    .rename(\"n_unique_simulants\")\n",
    "    .reset_index()\n",
    ")\n",
    "df_ops.compute(n_unique_simulants.n_unique_simulants.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a33ef1d-9690-43c9-9a78-cf2a6df8471e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pik_simulant_pairs = df_ops.persist(\n",
    "    pik_simulant_pairs.merge(\n",
    "        n_unique_simulants,\n",
    "        on=\"pik\",\n",
    "        how=\"left\",\n",
    "    )\n",
    ")\n",
    "pik_simulant_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5e4ed2b-12df-4a7a-9d6f-ad340050f540",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_ops.head(\n",
    "    pik_simulant_pairs[\n",
    "        pik_simulant_pairs.n_unique_simulants\n",
    "        == df_ops.compute(pik_simulant_pairs.n_unique_simulants.max())\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dcbd78c-a066-4884-ae54-639d3274e700",
   "metadata": {},
   "source": [
    "## Definitions of accuracy\n",
    "\n",
    "1. (most strict) Assigning any PIK with multiple simulants is incorrect\n",
    "2. Assigning a PIK with multiple simulants is neither incorrect nor correct (excluded from denominator)\n",
    "3. (most lenient) Assigning a PIK with multiple simulants is correct, as long as at least one of those simulants matches the truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d38a963-8f6c-4dad-99ed-be26005317c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# All modules, Medicare database, calculated from Layne, Wagner, and Rothhaas Table 1 (p. 15)\n",
    "real_life_pvs_accuracy = 1 - (2_585 + 60_709 + 129_480 + 89_094) / (\n",
    "    52_406_981 + 5_170_924 + 49_374_794 + 50_327_034\n",
    ")\n",
    "f\"{real_life_pvs_accuracy:.5%}\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19924c83-2aa2-4d4b-af94-b58f2f24c717",
   "metadata": {},
   "source": [
    "### Definition 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fbac00d-509f-487d-93cc-594fad36bc84",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "piks_assigned = df_ops.compute(census_2030_piked.pik.notnull().sum())\n",
    "piks_assigned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "982773c7-6147-4345-a912-24f80c61f76d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ops.head(pik_simulant_pairs[pik_simulant_pairs.n_unique_simulants > 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2602f38-0520-4ef0-abe0-93e536df78ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "single_sim_piks_correct = df_ops.compute(\n",
    "    census_2030_piked[[\"record_id\", \"pik\"]]\n",
    "    .merge(pik_simulant_pairs, on=\"pik\")\n",
    "    .merge(census_2030_ground_truth, on=\"record_id\")\n",
    "    .pipe(\n",
    "        lambda df: (df.simulant_id_x == df.simulant_id_y) & (df.n_unique_simulants == 1)\n",
    "    )\n",
    "    .sum()\n",
    ")\n",
    "single_sim_piks_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4a739ed-4e60-450c-8b53-f4bc7e930f8a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Overall accuracy, treating it as a black box\n",
    "(single_sim_piks_correct / piks_assigned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24bf6441-fe85-4045-a2ca-4349a7a6fe91",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assert len(confirmed_piks_with_ground_truth) == piks_assigned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11f17b3a-e2d2-4ccc-8fc9-cc1cc3e5fcb4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_ops.head(\n",
    "    census_2030_ground_truth.rename(columns={\"record_id\": \"record_id_census_2030\"})\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4963a01b-d07a-43b8-81b4-716ab79476a3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Looking at whether the exact *record* linked was from the same simulant\n",
    "single_sim_record_links_correct = df_ops.compute(\n",
    "    confirmed_piks_with_ground_truth.merge(\n",
    "        census_2030_ground_truth.rename(\n",
    "            columns={\"record_id\": \"record_id_raw_input_file\"}\n",
    "        ),\n",
    "        on=\"record_id_raw_input_file\",\n",
    "    )\n",
    "    .merge(\n",
    "        reference_files_ground_truth.rename(\n",
    "            columns={\"record_id\": \"record_id_reference_file\"}\n",
    "        ),\n",
    "        on=\"record_id_reference_file\",\n",
    "    )\n",
    "    .pipe(\n",
    "        lambda df: (df.simulant_id_x == df.simulant_id_y) & (df.n_unique_simulants == 1)\n",
    "    )\n",
    "    .sum()\n",
    ")\n",
    "single_sim_record_links_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ff66a85-851a-41b1-b9cd-5a625fcca0f9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "(single_sim_record_links_correct / piks_assigned)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93b6f302-6b80-477b-82ad-57a951b939e7",
   "metadata": {},
   "source": [
    "### Definition 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cedf1a5b-969a-4e7b-85c9-72650ecda7d1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "single_sim_piks_assigned = len(\n",
    "    census_2030_piked[[\"record_id\", \"pik\"]].merge(\n",
    "        pik_simulant_pairs[pik_simulant_pairs.n_unique_simulants == 1][\n",
    "            [\"pik\", \"simulant_id\"]\n",
    "        ]\n",
    "    )\n",
    ")\n",
    "single_sim_piks_assigned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf3f7251-a2d2-4123-b204-45e4a788acb5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Overall accuracy, treating it as a black box\n",
    "(single_sim_piks_correct / single_sim_piks_assigned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "926cf6e3-f704-4a32-ad93-a9552c17f0bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Looking at whether the exact *record* linked was from the same simulant\n",
    "single_sim_record_links_assigned = df_ops.compute(\n",
    "    (\n",
    "        confirmed_piks_with_ground_truth.merge(\n",
    "            reference_files_ground_truth.rename(\n",
    "                columns={\"record_id\": \"record_id_reference_file\"}\n",
    "            ),\n",
    "            on=\"record_id_reference_file\",\n",
    "        ).n_unique_simulants\n",
    "        == 1\n",
    "    ).sum()\n",
    ")\n",
    "single_sim_record_links_assigned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf705b2e-56a6-4393-9e5f-11f9198a1ce8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "(single_sim_record_links_correct / single_sim_record_links_assigned)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1030294-bd22-437c-827f-c8329b9703a9",
   "metadata": {},
   "source": [
    "### Definition 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a5ee70e-b466-4e02-b588-d0739a3cd055",
   "metadata": {},
   "outputs": [],
   "source": [
    "pik_simulant_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0782f353-0f5a-4887-a5df-0e127a1589d4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "piks_at_least_partially_correct = df_ops.persist(\n",
    "    census_2030_piked[[\"record_id\", \"pik\"]]\n",
    "    .merge(pik_simulant_pairs, on=\"pik\")\n",
    "    .merge(census_2030_ground_truth, on=\"record_id\")\n",
    "    .pipe(df_ops.drop_duplicates)\n",
    "    .assign(correct=lambda df: df.simulant_id_x == df.simulant_id_y)\n",
    "    .pipe(\n",
    "        df_ops.groupby_agg_small_groups,\n",
    "        by=[\"record_id\", \"pik\"],\n",
    "        agg_func=lambda x: x.correct.any(),\n",
    "    )\n",
    "    .reset_index()\n",
    ")\n",
    "piks_at_least_partially_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dd7ca2c-c4d5-480d-854d-38f6b4b50077",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Overall accuracy, treating it as a black box\n",
    "piks_correct_proportion = (\n",
    "    df_ops.compute(piks_at_least_partially_correct.correct.sum()) / piks_assigned\n",
    ")\n",
    "piks_correct_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a46c593-23fc-4919-be8c-2e31dfb6e5fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(\n",
    "    f\"{piks_correct_proportion:.5%} of the PIKs assigned were correct; compare with {real_life_pvs_accuracy:.5%} in real life\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e27befb-08e3-417a-8467-d0cd78936164",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Looking at whether the exact *record* linked was from the same simulant\n",
    "sim_record_links_at_least_partially_correct = df_ops.persist(\n",
    "    confirmed_piks_with_ground_truth.merge(\n",
    "        census_2030_ground_truth.rename(\n",
    "            columns={\"record_id\": \"record_id_raw_input_file\"}\n",
    "        ),\n",
    "        on=\"record_id_raw_input_file\",\n",
    "    )\n",
    "    .merge(\n",
    "        reference_files_ground_truth.rename(\n",
    "            columns={\"record_id\": \"record_id_reference_file\"}\n",
    "        ),\n",
    "        on=\"record_id_reference_file\",\n",
    "    )\n",
    "    .assign(correct=lambda df: df.simulant_id_x == df.simulant_id_y)\n",
    "    .pipe(\n",
    "        df_ops.groupby_agg_small_groups,\n",
    "        by=[\n",
    "            \"record_id_raw_input_file\",\n",
    "            \"record_id_reference_file\",\n",
    "            \"pik\",\n",
    "            \"module_name\",\n",
    "            \"pass_name\",\n",
    "        ],\n",
    "        agg_func=lambda x: x.correct.any(),\n",
    "    )\n",
    "    .reset_index()\n",
    ")\n",
    "sim_record_links_at_least_partially_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e36b423d-ad09-4947-bf5b-748f1fd04b8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(sim_record_links_at_least_partially_correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37debec5-791e-4854-84c3-dc9339860555",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(\n",
    "    df_ops.drop_duplicates(\n",
    "        sim_record_links_at_least_partially_correct[\n",
    "            [\"record_id_raw_input_file\", \"record_id_reference_file\"]\n",
    "        ]\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "088af209-ccdc-48c7-91dc-ff7a78e9f2d1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "(\n",
    "    df_ops.compute(sim_record_links_at_least_partially_correct.correct.sum())\n",
    "    / piks_assigned\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29a172f9-0197-4193-99fd-0eaba0ac30e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "assert df_ops.compute(\n",
    "    (\n",
    "        df_ops.groupby_agg_small_groups(\n",
    "            confirmed_piks_with_ground_truth,\n",
    "            by=\"record_id_raw_input_file\",\n",
    "            agg_func=lambda x: x.record_id_reference_file.nunique(),\n",
    "        )\n",
    "        <= 1\n",
    "    ).all()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e4b30e7-3164-449d-b333-00a1c2faae8d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Using definition 3 -- at the PIK level\n",
    "piks_at_least_partially_correct = df_ops.persist(\n",
    "    piks_at_least_partially_correct.rename(\n",
    "        columns={\"record_id\": \"record_id_raw_input_file\"}\n",
    "    ).merge(\n",
    "        confirmed_piks_with_ground_truth[\n",
    "            [\"record_id_raw_input_file\", \"module_name\", \"pass_name\"]\n",
    "        ],\n",
    "        on=\"record_id_raw_input_file\",\n",
    "    )\n",
    ")\n",
    "piks_at_least_partially_correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5bf5cf5-8be6-4b8c-a9cb-60b8e8b9fd7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Accuracy by module -- note that this shows the opposite pattern (with the sample data)\n",
    "# relative to the results of Layne et al., who found GeoSearch was much *more* accurate\n",
    "df_ops.compute(\n",
    "    piks_at_least_partially_correct.groupby(\"module_name\")\n",
    "    .correct.agg([\"mean\", \"size\"])\n",
    "    .sort_values(\"mean\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17cf823e-9e07-43f4-972a-65b960d204cc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Accuracy by pass -- could be used to tune pass-specific cutoffs, but\n",
    "# this might not be too informative while we are still using the sample data.\n",
    "df_ops.compute(\n",
    "    piks_at_least_partially_correct.groupby([\"module_name\", \"pass_name\"])\n",
    "    .correct.agg([\"mean\", \"size\"])\n",
    "    .sort_values(\"mean\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "001b89a0-1db1-4442-878e-3472cce4c40a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Using definition 3 -- at the link level\n",
    "df_ops.compute(\n",
    "    sim_record_links_at_least_partially_correct.groupby(\"module_name\")\n",
    "    .correct.agg([\"mean\", \"size\"])\n",
    "    .sort_values(\"mean\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "491fee9a-d01a-4b39-8023-d5e57f9e0ae9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_ops.compute(\n",
    "    sim_record_links_at_least_partially_correct.groupby([\"module_name\", \"pass_name\"])\n",
    "    .correct.agg([\"mean\", \"size\"])\n",
    "    .sort_values(\"mean\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25dbd840-d252-42ce-9eac-b38fabf20928",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ops.compute(\n",
    "    sim_record_links_at_least_partially_correct[\n",
    "        ~sim_record_links_at_least_partially_correct.correct\n",
    "    ]\n",
    "    .groupby([\"module_name\", \"pass_name\"])\n",
    "    .size()\n",
    ").sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ced1468-6ad9-4ca5-b6d9-94ab29bd1e42",
   "metadata": {},
   "source": [
    "### Incorrect and missed PIKs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6e24720-b1b3-4c9e-a1be-bfe33aa2d635",
   "metadata": {},
   "outputs": [],
   "source": [
    "incorrectly_linked_pairs = df_ops.persist(\n",
    "    df_ops.drop_duplicates(\n",
    "        sim_record_links_at_least_partially_correct[\n",
    "            ~sim_record_links_at_least_partially_correct.correct\n",
    "        ][[\"record_id_raw_input_file\", \"record_id_reference_file\"]]\n",
    "    )\n",
    ")\n",
    "incorrectly_linked_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6baf95ac-5a58-43e3-b8f5-e20b975dd26c",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(incorrectly_linked_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ba24dba-612f-49fb-94c9-e86c1c07b3de",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "incorrect_links = df_ops.head(incorrectly_linked_pairs, n=100)\n",
    "incorrect_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28c753cd-d01a-4eaa-adfa-0ee583f001e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "%xdel incorrectly_linked_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4337a127-681d-4ce1-9b91-1bdd86dfcaab",
   "metadata": {},
   "outputs": [],
   "source": [
    "comparison_cols = [\n",
    "    \"first_name\",\n",
    "    \"middle_name\",\n",
    "    \"last_name\",\n",
    "    \"date_of_birth\",\n",
    "    \"street_number\",\n",
    "    \"street_name\",\n",
    "    \"unit_number\",\n",
    "    \"city\",\n",
    "    \"state\",\n",
    "]\n",
    "\n",
    "incorrect_links_detail = incorrect_links.merge(\n",
    "    df_ops.compute(\n",
    "        census_2030_piked[\n",
    "            census_2030_piked.record_id.isin(incorrect_links.record_id_raw_input_file)\n",
    "        ]\n",
    "    ).rename(\n",
    "        columns={\n",
    "            \"record_id\": \"record_id_raw_input_file\",\n",
    "            \"middle_initial\": \"middle_name\",\n",
    "        }\n",
    "    )[\n",
    "        [\"record_id_raw_input_file\"] + comparison_cols\n",
    "    ],\n",
    "    on=\"record_id_raw_input_file\",\n",
    "    how=\"left\",\n",
    ").merge(\n",
    "    df_ops.compute(\n",
    "        reference_file[\n",
    "            reference_file.record_id.isin(incorrect_links.record_id_reference_file)\n",
    "        ]\n",
    "    )\n",
    "    .rename(columns={\"record_id\": \"record_id_reference_file\"})\n",
    "    .rename(columns=lambda c: c.replace(\"mailing_address_\", \"\"))[\n",
    "        [\"record_id_reference_file\"] + comparison_cols\n",
    "    ],\n",
    "    on=\"record_id_reference_file\",\n",
    "    how=\"left\",\n",
    "    suffixes=(\"_census\", \"_reference_file\"),\n",
    ")\n",
    "\n",
    "\n",
    "def flatten(xss):\n",
    "    return [x for xs in xss for x in xs]\n",
    "\n",
    "\n",
    "incorrect_links_detail[\n",
    "    flatten([(f\"{c}_census\", f\"{c}_reference_file\") for c in comparison_cols])\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d1db69a-4c15-4455-a563-098f65125a0c",
   "metadata": {
    "papermill": {
     "duration": 0.125926,
     "end_time": "2024-05-08T19:32:45.873557",
     "exception": false,
     "start_time": "2024-05-08T19:32:45.747631",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "missed_links = df_ops.persist(\n",
    "    census_2030_piked[census_2030_piked.pik.isnull()][[\"record_id\"]]\n",
    "    .merge(census_2030_ground_truth, on=\"record_id\")\n",
    "    .merge(\n",
    "        reference_files_ground_truth[\n",
    "            reference_files_ground_truth.n_unique_simulants == 1\n",
    "        ],\n",
    "        on=\"simulant_id\",\n",
    "        suffixes=(\"_census\", \"_reference_file\"),\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cecb6388-2be3-4da5-a3b9-48fd65f9fd3a",
   "metadata": {
    "papermill": {
     "duration": 72.408287,
     "end_time": "2024-05-08T19:36:37.163556",
     "exception": false,
     "start_time": "2024-05-08T19:35:24.755269",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "len(missed_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05a5f937-a513-4624-9c60-2ca1e1ed4b85",
   "metadata": {
    "papermill": {
     "duration": 1.193872,
     "end_time": "2024-05-08T19:36:38.377398",
     "exception": false,
     "start_time": "2024-05-08T19:36:37.183526",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "simulants_missed = df_ops.head(\n",
    "    missed_links[[\"simulant_id\"]], n=100\n",
    ").simulant_id.unique()\n",
    "simulants_missed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20658db7-1119-4671-8468-4def514df8d7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "missed_pairs = df_ops.compute(\n",
    "    missed_links[missed_links.simulant_id.isin(list(simulants_missed))]\n",
    ")\n",
    "missed_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "368bda61-88af-4521-929c-9fa9e71a67c3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%xdel missed_links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1026714-47d0-4263-9cd4-f65ea5a0f882",
   "metadata": {
    "papermill": {
     "duration": 0.245016,
     "end_time": "2024-05-08T19:36:38.650379",
     "exception": false,
     "start_time": "2024-05-08T19:36:38.405363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "missed_links_detail = missed_pairs.merge(\n",
    "    df_ops.compute(\n",
    "        census_2030_piked[\n",
    "            census_2030_piked.record_id.isin(list(missed_pairs.record_id_census))\n",
    "        ]\n",
    "    ).rename(\n",
    "        columns={\"record_id\": \"record_id_census\", \"middle_initial\": \"middle_name\"}\n",
    "    ),\n",
    "    on=\"record_id_census\",\n",
    ").merge(\n",
    "    df_ops.compute(\n",
    "        reference_file[\n",
    "            reference_file.record_id.isin(missed_pairs.record_id_reference_file)\n",
    "        ]\n",
    "    )\n",
    "    .rename(columns=lambda c: c.replace(\"mailing_address_\", \"\"))\n",
    "    .rename(columns={\"record_id\": \"record_id_reference_file\"}),\n",
    "    on=\"record_id_reference_file\",\n",
    "    suffixes=(\"_census\", \"_reference_file\"),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7f0e3d2-ea2f-4ba9-a66e-ed3a315a4004",
   "metadata": {
    "papermill": {
     "duration": 8231.679645,
     "end_time": "2024-05-08T21:53:50.360774",
     "exception": true,
     "start_time": "2024-05-08T19:36:38.681129",
     "status": "failed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for simulant in simulants_missed:\n",
    "    print(simulant)\n",
    "    display(\n",
    "        missed_links_detail[missed_links_detail.simulant_id == simulant][\n",
    "            [\"simulant_id\"]\n",
    "            + flatten([(f\"{c}_census\", f\"{c}_reference_file\") for c in comparison_cols])\n",
    "        ]\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5df08684-940c-4860-8fc6-e89021a6c1dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "person_linkage_case_study_20240423",
   "language": "python",
   "name": "person_linkage_case_study_20240423"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}