homework3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "53941c95cdf94f468aaabeace1e3e187": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_509f4fe84c0d4d3098f2dc3069db3631",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a8d35f831b044f3d8152235c510a59b7",
            "value": 0
          }
        },
        "509f4fe84c0d4d3098f2dc3069db3631": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "a8d35f831b044f3d8152235c510a59b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5f35dfe1662846abae4221a6a99e876a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b35960aca1f48faa096305703963fb7",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_431cf91bef1e49fbb4185076fa878f2a",
            "value": 0
          }
        },
        "8b35960aca1f48faa096305703963fb7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "431cf91bef1e49fbb4185076fa878f2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7c974b5a06b14b1cb81c32706ee88cd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c888352c732d45619eeb3a99b98a02fe",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_50b7c0fd9d36464ca8f4cd6f780592f3",
            "value": 0
          }
        },
        "c888352c732d45619eeb3a99b98a02fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "50b7c0fd9d36464ca8f4cd6f780592f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e1f62c4d1df241c7ad24fe22ef01ff55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf932f6db5324206b7eb565a4e1baef4",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c112235b3f4f45ff93f8fce293526e6f",
            "value": 0
          }
        },
        "bf932f6db5324206b7eb565a4e1baef4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "c112235b3f4f45ff93f8fce293526e6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a1340938cc2c4a3db3fcb3527479cc8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ff2570d76eea4a8fadece45387460ca0",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9088a788aef742269e25a6780c2e1847",
            "value": 0
          }
        },
        "ff2570d76eea4a8fadece45387460ca0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "9088a788aef742269e25a6780c2e1847": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c3abe6dfac684b8aad2532fdbf9457ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d3bfb554be440108560cea488dace9f",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_472e72e23fa04378882464407dea402d",
            "value": 0.7142857142857143
          }
        },
        "1d3bfb554be440108560cea488dace9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "472e72e23fa04378882464407dea402d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6da280f9b05248a8b677e49757559957": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7b0dcb00efdb4e4db15759d4a9c8c241",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c4d54790c85d443393fd6ffb26e512d2",
            "value": 0
          }
        },
        "7b0dcb00efdb4e4db15759d4a9c8c241": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "c4d54790c85d443393fd6ffb26e512d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0ea241ae625b465b8bbb94ca7b755469": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_09ddd01e844941fa825abc8249b7bf43",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ad7be9f0560d4ce1a805d392c3f0fc73",
            "value": 0.8888888888888888
          }
        },
        "09ddd01e844941fa825abc8249b7bf43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "ad7be9f0560d4ce1a805d392c3f0fc73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "93f5cfb8aec64cd09de0383d6c9cb32e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42d26ad24a7940978850ea4d34a22f92",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0c88f1a77ad140728781e85d7d5a727b",
            "value": 0
          }
        },
        "42d26ad24a7940978850ea4d34a22f92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "0c88f1a77ad140728781e85d7d5a727b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "01736961e1474b3ba131162f242a2664": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_470944545c534a2f85b4fca11efe6568",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0dea508687d14f3d984fa0f3e7244aab",
            "value": 0
          }
        },
        "470944545c534a2f85b4fca11efe6568": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "0dea508687d14f3d984fa0f3e7244aab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5ea868974d914e4295dbad2b192168f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc32fa60b62e452b94034d71f723fe51",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_135a21cb249843eb92eab7e8074691fa",
            "value": 0.1111111111111111
          }
        },
        "dc32fa60b62e452b94034d71f723fe51": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "135a21cb249843eb92eab7e8074691fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6cb675ba10ff46929fcda37cfea8c1c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_51f9861db1b54458ad6ca8dfe28c47eb",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_11ea21142ac84ba4ad174635856cdab1",
            "value": 0
          }
        },
        "51f9861db1b54458ad6ca8dfe28c47eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "11ea21142ac84ba4ad174635856cdab1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0075b5ac83fb4f119f593f6c8470e9e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f41d88f46874e3d814335e7453f1b1e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7755c1b7f04d44bbba47650676fe1ab1",
            "value": 0
          }
        },
        "6f41d88f46874e3d814335e7453f1b1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "7755c1b7f04d44bbba47650676fe1ab1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "089bd7251f1a430380d1b3f58382c01b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b3c995aa2fdf4d759239d1bd7f74b215",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4a784f0c852d4dd0aaf7cdc92cab1fa6",
            "value": 0.8888888888888888
          }
        },
        "b3c995aa2fdf4d759239d1bd7f74b215": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "4a784f0c852d4dd0aaf7cdc92cab1fa6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6820f35f0403466a99813587a84fdfe8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b80833acd2d04441819e2d1b8971291f",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5e5b26e4911c42d78614c1472b122b20",
            "value": 0
          }
        },
        "b80833acd2d04441819e2d1b8971291f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "5e5b26e4911c42d78614c1472b122b20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3cc898c09eca4a2dbb75eff34c03668d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b92ea60ea6a848bc862ce1f544da5e0e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4799c55f47d64ce5a1e5c2f157592103",
            "value": 0
          }
        },
        "b92ea60ea6a848bc862ce1f544da5e0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "4799c55f47d64ce5a1e5c2f157592103": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ad0e3b506276483ea11aaab9d998f9ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9308e999af06414ea6d9a861157992d5",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3537a8712ade4c81bb2cbbb7f523ac91",
            "value": 0
          }
        },
        "9308e999af06414ea6d9a861157992d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "3537a8712ade4c81bb2cbbb7f523ac91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b2ff07681ace472899bee8aff4d6038f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_91c8452eaa134ac09460c6b4bcc977a7",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0f04f931589f4cd5a04d4905deaef2cb",
            "value": 0
          }
        },
        "91c8452eaa134ac09460c6b4bcc977a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "0f04f931589f4cd5a04d4905deaef2cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "706102b434e14e62b34668503972784b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2000ac6d2ded45aebc86b000f40810ba",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b742074707ff443380530b6786c77a6f",
            "value": 0
          }
        },
        "2000ac6d2ded45aebc86b000f40810ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "b742074707ff443380530b6786c77a6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cd502149295243cc81f65c7988a02331": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bbbd324f47674fe0ae49f103696de8cd",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_69d0aeefbdbe4c33950bcfd848a5852e",
            "value": 0
          }
        },
        "bbbd324f47674fe0ae49f103696de8cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "69d0aeefbdbe4c33950bcfd848a5852e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "323a668fa3d04097b7b90fcd1f575705": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0afab056c19847f38ec5687d887a5a47",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9ac634d36ec74968b15bae732a643b97",
            "value": 0
          }
        },
        "0afab056c19847f38ec5687d887a5a47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "9ac634d36ec74968b15bae732a643b97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bd880327728749eeba445d1f819a1e04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a95e5cb8ef549838254729e399b5f9d",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7f22f00142fe40a78a69c5942bebbf1d",
            "value": 0
          }
        },
        "9a95e5cb8ef549838254729e399b5f9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "7f22f00142fe40a78a69c5942bebbf1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9e6cb707b4a9443e8bfe09ff8aecb401": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f855fd4dd9a74e7ca958fb8ea6b4cf0b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6d19c25209ab494983c9442552276d55",
            "value": 0
          }
        },
        "f855fd4dd9a74e7ca958fb8ea6b4cf0b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "6d19c25209ab494983c9442552276d55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0d6c2f1430ef4382896116fe4eacbb26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9dd7aa464b584daa81a8889281f0c3bd",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c2d0fc4b55c04e50baace1715d0b8476",
            "value": 0
          }
        },
        "9dd7aa464b584daa81a8889281f0c3bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "c2d0fc4b55c04e50baace1715d0b8476": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "734e83870e8845d696fcbe8132a375f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_23124827d61141139e4f5f37f72a01b0",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e589d85e64564bc096829ff49354d911",
            "value": 0
          }
        },
        "23124827d61141139e4f5f37f72a01b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "e589d85e64564bc096829ff49354d911": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "976b8a287dc14898a17236f13afad526": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_abae0496d8b848efb1c7f265944f34a2",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_32e506a7998a4f79b31dd805306d1b5c",
            "value": 0
          }
        },
        "abae0496d8b848efb1c7f265944f34a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "32e506a7998a4f79b31dd805306d1b5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "65a8fe86356146bda357f13d5348b3ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f0876230fd3477daf4c35af81bb5b65",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c632002eb3ba44babc27a16ea4b9d13c",
            "value": 0
          }
        },
        "1f0876230fd3477daf4c35af81bb5b65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "c632002eb3ba44babc27a16ea4b9d13c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0b2deaa1b6714b5f8df127076c8fc080": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_536d1266c0404a0b82ecb7b27afaeb2a",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_32cc6f6c35fb44638d46ae15147f8186",
            "value": 0
          }
        },
        "536d1266c0404a0b82ecb7b27afaeb2a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "32cc6f6c35fb44638d46ae15147f8186": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7bafb8c72e9d44eebf5a2aff6b33ae56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85e783308f604aa88675583bf05e18da",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f6b80c7e17304cc5a56797950ddd7835",
            "value": 0
          }
        },
        "85e783308f604aa88675583bf05e18da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "f6b80c7e17304cc5a56797950ddd7835": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8770d9b7a75541cc989297353fed2475": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9df659ee2edf400682c631dbc7027ee4",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_366f2bb2d61d498183f1d4c1139007bc",
            "value": 0
          }
        },
        "9df659ee2edf400682c631dbc7027ee4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "366f2bb2d61d498183f1d4c1139007bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b2d9014bdc2646e2a43acce3e4f7dd82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89176493cffa4e8dbb892b1ef35d8868",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_956d03b516ae40a49837182cf1fcdba0",
            "value": 0
          }
        },
        "89176493cffa4e8dbb892b1ef35d8868": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "956d03b516ae40a49837182cf1fcdba0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "836b4fab1c9a41e498e7c7b7079159e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_295f58937e17427fb74d976de20998a7",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_041d3e2705b54d4391c88a9acef9341d",
            "value": 0.9619771863117871
          }
        },
        "295f58937e17427fb74d976de20998a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "041d3e2705b54d4391c88a9acef9341d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b8c35410b9fb4b12b47e12fa3c04facc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc616afaf0354b3f8b3cccccb1c2904b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_37acc7d113d64fb3b284a380593bd669",
            "value": 0
          }
        },
        "cc616afaf0354b3f8b3cccccb1c2904b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "37acc7d113d64fb3b284a380593bd669": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "13fd488a60794829a54da03f4bf20eb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7fd40aa49a4a40ea9bdc1e4fa607a9b1",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_79f3ed2b74c2411ba71aad6394618002",
            "value": 0
          }
        },
        "7fd40aa49a4a40ea9bdc1e4fa607a9b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "79f3ed2b74c2411ba71aad6394618002": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "764ee3d6b0a7412e89db41165840066f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4912477113c646a1909220eeb5cde010",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f57987a0052b4be29fb22a1d51e1504f",
            "value": 0.9372294372294372
          }
        },
        "4912477113c646a1909220eeb5cde010": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "f57987a0052b4be29fb22a1d51e1504f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0fa1b4ff9c547fba6a3b7fe758a6230": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97a93b09ebb54368a3c0531624b1412d",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_df67d3f890854fdcbc994691b9c05ad2",
            "value": 0
          }
        },
        "97a93b09ebb54368a3c0531624b1412d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "df67d3f890854fdcbc994691b9c05ad2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "719d1dafdb2e405aa3b61583db8bac41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed56f99bf66a4db8a113531b4c2e6c96",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5523fc2c85234866a430afced7d41261",
            "value": 0
          }
        },
        "ed56f99bf66a4db8a113531b4c2e6c96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "5523fc2c85234866a430afced7d41261": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3950f444830440a786757848cfc5c288": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_653aa1db75a24854bf74ad11532f7fa7",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b16ac7fe3b3941c8a0b8846068ab1632",
            "value": 0.9959514170040485
          }
        },
        "653aa1db75a24854bf74ad11532f7fa7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "b16ac7fe3b3941c8a0b8846068ab1632": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "24710dbd5dc6429a8754a034c62d0687": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_88cc5b9fd4f740e0b57d488457a7dae5",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cdfd0276bb3a49b9aaddc555483d1f47",
            "value": 0.5669291338582677
          }
        },
        "88cc5b9fd4f740e0b57d488457a7dae5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "cdfd0276bb3a49b9aaddc555483d1f47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5cb8987a8cb84302aa7a8da3ffcc021a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f19a6074b2d3460b89ac482761a086f8",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_001fc24932214ee9960d98a3788a2730",
            "value": 0
          }
        },
        "f19a6074b2d3460b89ac482761a086f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "001fc24932214ee9960d98a3788a2730": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3b4a83d309b645cf96c8fc2fab09be36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b44bd77dda324e8b90277725743c6a41",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1c0b612fc0a548c3a8b23711852153b7",
            "value": 0.4825018615040953
          }
        },
        "b44bd77dda324e8b90277725743c6a41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "1c0b612fc0a548c3a8b23711852153b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ef15c05e2e2a4f73b60656712e44fa7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_492cfaee7ae74661bad025b03ebaec85",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9ee4b47f40c84691b02aa247a7330b28",
            "value": 0.967741935483871
          }
        },
        "492cfaee7ae74661bad025b03ebaec85": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "9ee4b47f40c84691b02aa247a7330b28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4691a7f37bba41ddb8e6a6d8061efb4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e20ba4bb50543b9a4f11da4225c2282",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cfc424af93334191a4ae169a8e8b2b36",
            "value": 0
          }
        },
        "4e20ba4bb50543b9a4f11da4225c2282": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "cfc424af93334191a4ae169a8e8b2b36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6cb4268f54b14ffea28d30b83f0b8d55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_133e9e12e1c84f3abcbf623d8219bb6b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6b1ee1fcba2c4331a9bd30a39cf8b6f9",
            "value": 0.5670995670995671
          }
        },
        "133e9e12e1c84f3abcbf623d8219bb6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "6b1ee1fcba2c4331a9bd30a39cf8b6f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4d360fe1aa8c47a9b78ccd50ad4dad91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9cd9f0d5314488c91c794176555361e",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9a6a6bd8b93349dfab569ff2d640d049",
            "value": 0.5
          }
        },
        "e9cd9f0d5314488c91c794176555361e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "9a6a6bd8b93349dfab569ff2d640d049": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5d40532a2626405ba08a5485578d012f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f77ee9adde974f8791c38c5e8a5dd625",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bdb4a1b2346c40138656aab3ba5de232",
            "value": 0
          }
        },
        "f77ee9adde974f8791c38c5e8a5dd625": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "bdb4a1b2346c40138656aab3ba5de232": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c97a0104645049ce97098f682e0d1025": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94f5e3516e3a4dc480fc6bb16107140f",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9c5192d9c7d14c1eaca3e1e3e5357064",
            "value": 0.8250950570342205
          }
        },
        "94f5e3516e3a4dc480fc6bb16107140f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "9c5192d9c7d14c1eaca3e1e3e5357064": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2370ecdf88ec4cf3b4e7e67465144936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_126a3bae93ba4b3ba4572d87c35f1ee3",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8770a0b180374a35be3f0372f7db3127",
            "value": 0.5727272727272728
          }
        },
        "126a3bae93ba4b3ba4572d87c35f1ee3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "8770a0b180374a35be3f0372f7db3127": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "effb549629cf4eb4aa46cbd838abe009": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d1950bdaa7a4e73a42344ba9efbedfc",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b158dd2e6a574eb6a78d224c5b1ace69",
            "value": 0
          }
        },
        "5d1950bdaa7a4e73a42344ba9efbedfc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "b158dd2e6a574eb6a78d224c5b1ace69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8bbcab2efd8d4f5ba2c2b07c8caf76b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8dffb39a819744e188af492d25cae2dd",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f805b139654d4bdeadd95473ec9505d8",
            "value": 0.9959514170040485
          }
        },
        "8dffb39a819744e188af492d25cae2dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "f805b139654d4bdeadd95473ec9505d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2c824839e7724b6da705fac746adf6dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69afee623f99482aa3fc888adceaccb9",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_03f53a716ab84aef95c759e3ee6987bb",
            "value": 0.9193548387096774
          }
        },
        "69afee623f99482aa3fc888adceaccb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "03f53a716ab84aef95c759e3ee6987bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "aa5ca98974bb46f4bce724e44d837396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85758bc889d24c4a9c105cc22e4ca1e8",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8d9a3445370e460aa462dc44f14cd22b",
            "value": 0
          }
        },
        "85758bc889d24c4a9c105cc22e4ca1e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "8d9a3445370e460aa462dc44f14cd22b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5b9901221c6e4e889cfa91e8859549e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "Progress:",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c7db6b275d6043638e6811177904d59c",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_215450bc5f1f475198001f3d8432561f",
            "value": 0.31798806479113384
          }
        },
        "c7db6b275d6043638e6811177904d59c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": "25px",
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "215450bc5f1f475198001f3d8432561f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qVMCjEV1IEC7"
      },
      "source": [
        "# CIS 545 Homework 3: Spark SQL\n",
        "\n",
        "#### **Worth 100 points, due Tuesday November 1st, 10:00 PM EST**\n",
        "\n",
        "Welcome to CIS 545 Homework 3! In this homework you will master Spark SQL. By the end, you'll be a star (not that you aren't already one). Over the next few days you will be using an EMR cluster to use Spark to manipulate the datasets about Amazon products and their reviews. \n",
        "\n",
        "## The Necessary Notes and Nags\n",
        "Before we begin here are some important notes to keep in mind,\n",
        "\n",
        "\n",
        "1.   **IMPORTANT!** I said it twice, it's really important. In this homework, we will be using AWS resources. You are given a quota ($100) to use for the entirety of the homework. There is a small chance you will use all this money, however it is important that at the end of every session, you **shut down your EMR cluster**.\n",
        "2.   **Be sure you use Google Colab for this Homework** since we must connect to the EMR cluster and local Jupyter will have issues doing that. Using a Google Colab Notebook with an EMR cluster has two important abnormalities:\n",
        "    * The first line of any cell in which you will use the spark session must be `%%spark`. Notice that all cells below have this.\n",
        "    * You will, unfortunately, not be able to stop a cell while it is running. If you wish to do so, you will need to restart your cluster. See the Setup EMR Document for reference.\n",
        "3.   You are **required** to use Spark SQL queries to handle the data in the assignment. Mastering SQL is more beneficial than being able to use Spark commands (functions) as it will show up in more areas of programming and data science/analytics than just Spark. Use the following [function list](https://spark.apache.org/docs/latest/api/sql/index.html#) to see all the SQL functions avaliable in Spark.\n",
        "4.   Throughout the homework you will be manipulating Spark dataframes (sdfs). We do not specify any ordering on the final output. You are welcome to order your final tables in whatever way you deem fit. We will conduct our own ordering when we grade.\n",
        "5. Based on the challenges you've faced in the previous homework, we are including information on the expected schema of your results.  Apache Spark is very fiddly but we hope this will help.\n",
        "6. There are portions of this homework that are _very_ hard. We urge you start early to come to office hours and get help if you get stuck. But don't worry, I can see the future, and you've all got this.\n",
        "\n",
        "With that said, let's dive in.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7XEqGpEGBWs5"
      },
      "source": [
        "## Step 0: Set up EMR\n",
        "\n",
        "Follow the [AWS Academy Getting Started](https://docs.google.com/document/d/1JPitLGaorjTbXjGsaoIHcLTu2cj8rjm5UNr9bSpZ72k/edit?usp=sharing) instructions.\n",
        "\n",
        "Move on to Step 0.1 after you have completed all the steps in the document.\n",
        "\n",
        "![ACME GIANT RUBBER BAND](https://pbs.twimg.com/media/DRqbJh7UMAE2z4o?format=jpg&name=4096x4096)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5iBPXxgAdXkv"
      },
      "source": [
        "### Step 0.1: The Superfluous Setup\n",
        "\n",
        "Run the following two cells. These will allow your colab notebook to connect to an use your EMR."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pvkEbVaaAQ1e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "324844b9-3f4f-495c-ae91-eb1455d2efaa"
      },
      "source": [
        "#%%capture\n",
        "!apt install libkrb5-dev\n",
        "!pip install sparkmagic\n",
        "!pip install -i https://test.pypi.org/simple/ penn-grader==0.5.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  comerr-dev krb5-multidev libgssrpc4 libkadm5clnt-mit11 libkadm5srv-mit11\n",
            "  libkdb5-9\n",
            "Suggested packages:\n",
            "  doc-base krb5-doc krb5-user\n",
            "The following NEW packages will be installed:\n",
            "  comerr-dev krb5-multidev libgssrpc4 libkadm5clnt-mit11 libkadm5srv-mit11\n",
            "  libkdb5-9 libkrb5-dev\n",
            "0 upgraded, 7 newly installed, 0 to remove and 4 not upgraded.\n",
            "Need to get 349 kB of archives.\n",
            "After this operation, 1,992 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libgssrpc4 amd64 1.16-2ubuntu0.2 [54.4 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libkdb5-9 amd64 1.16-2ubuntu0.2 [37.3 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libkadm5srv-mit11 amd64 1.16-2ubuntu0.2 [49.7 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libkadm5clnt-mit11 amd64 1.16-2ubuntu0.2 [38.0 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 comerr-dev amd64 2.1-1.44.1-1ubuntu1.4 [38.5 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 krb5-multidev amd64 1.16-2ubuntu0.2 [120 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libkrb5-dev amd64 1.16-2ubuntu0.2 [11.7 kB]\n",
            "Fetched 349 kB in 1s (404 kB/s)\n",
            "Selecting previously unselected package libgssrpc4:amd64.\n",
            "(Reading database ... 123941 files and directories currently installed.)\n",
            "Preparing to unpack .../0-libgssrpc4_1.16-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking libgssrpc4:amd64 (1.16-2ubuntu0.2) ...\n",
            "Selecting previously unselected package libkdb5-9:amd64.\n",
            "Preparing to unpack .../1-libkdb5-9_1.16-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking libkdb5-9:amd64 (1.16-2ubuntu0.2) ...\n",
            "Selecting previously unselected package libkadm5srv-mit11:amd64.\n",
            "Preparing to unpack .../2-libkadm5srv-mit11_1.16-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking libkadm5srv-mit11:amd64 (1.16-2ubuntu0.2) ...\n",
            "Selecting previously unselected package libkadm5clnt-mit11:amd64.\n",
            "Preparing to unpack .../3-libkadm5clnt-mit11_1.16-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking libkadm5clnt-mit11:amd64 (1.16-2ubuntu0.2) ...\n",
            "Selecting previously unselected package comerr-dev:amd64.\n",
            "Preparing to unpack .../4-comerr-dev_2.1-1.44.1-1ubuntu1.4_amd64.deb ...\n",
            "Unpacking comerr-dev:amd64 (2.1-1.44.1-1ubuntu1.4) ...\n",
            "Selecting previously unselected package krb5-multidev:amd64.\n",
            "Preparing to unpack .../5-krb5-multidev_1.16-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking krb5-multidev:amd64 (1.16-2ubuntu0.2) ...\n",
            "Selecting previously unselected package libkrb5-dev:amd64.\n",
            "Preparing to unpack .../6-libkrb5-dev_1.16-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking libkrb5-dev:amd64 (1.16-2ubuntu0.2) ...\n",
            "Setting up libgssrpc4:amd64 (1.16-2ubuntu0.2) ...\n",
            "Setting up comerr-dev:amd64 (2.1-1.44.1-1ubuntu1.4) ...\n",
            "Setting up libkdb5-9:amd64 (1.16-2ubuntu0.2) ...\n",
            "Setting up libkadm5srv-mit11:amd64 (1.16-2ubuntu0.2) ...\n",
            "Setting up libkadm5clnt-mit11:amd64 (1.16-2ubuntu0.2) ...\n",
            "Setting up krb5-multidev:amd64 (1.16-2ubuntu0.2) ...\n",
            "Setting up libkrb5-dev:amd64 (1.16-2ubuntu0.2) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.6) ...\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sparkmagic\n",
            "  Downloading sparkmagic-0.20.0.tar.gz (44 kB)\n",
            "\u001b[K     |████████████████████████████████| 44 kB 862 kB/s \n",
            "\u001b[?25hCollecting hdijupyterutils>=0.6\n",
            "  Downloading hdijupyterutils-0.20.0.tar.gz (5.0 kB)\n",
            "Collecting autovizwidget>=0.6\n",
            "  Downloading autovizwidget-0.20.0.tar.gz (8.8 kB)\n",
            "Requirement already satisfied: ipython>=5 in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (7.9.0)\n",
            "Collecting nose\n",
            "  Downloading nose-1.3.7-py3-none-any.whl (154 kB)\n",
            "\u001b[K     |████████████████████████████████| 154 kB 11.7 MB/s \n",
            "\u001b[?25hCollecting mock\n",
            "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Requirement already satisfied: pandas>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (1.3.5)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (1.21.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (2.23.0)\n",
            "Requirement already satisfied: ipykernel>=4.2.2 in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (5.3.4)\n",
            "Requirement already satisfied: ipywidgets>5.0.0 in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (7.7.1)\n",
            "Requirement already satisfied: notebook>=4.2 in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (5.7.16)\n",
            "Requirement already satisfied: tornado>=4 in /usr/local/lib/python3.7/dist-packages (from sparkmagic) (6.0.4)\n",
            "Collecting requests_kerberos>=0.8.0\n",
            "  Downloading requests_kerberos-0.14.0-py2.py3-none-any.whl (11 kB)\n",
            "Collecting nest_asyncio==1.5.5\n",
            "  Downloading nest_asyncio-1.5.5-py3-none-any.whl (5.2 kB)\n",
            "Requirement already satisfied: plotly>=3 in /usr/local/lib/python3.7/dist-packages (from autovizwidget>=0.6->sparkmagic) (5.5.0)\n",
            "Collecting jupyter>=1\n",
            "  Downloading jupyter-1.0.0-py2.py3-none-any.whl (2.7 kB)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel>=4.2.2->sparkmagic) (6.1.12)\n",
            "Requirement already satisfied: traitlets>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel>=4.2.2->sparkmagic) (5.1.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (2.0.10)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (2.6.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (4.4.2)\n",
            "Collecting jedi>=0.10\n",
            "  Downloading jedi-0.18.1-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 46.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (0.7.5)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (57.4.0)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (4.8.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.7/dist-packages (from ipython>=5->sparkmagic) (0.2.0)\n",
            "Requirement already satisfied: widgetsnbextension~=3.6.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets>5.0.0->sparkmagic) (3.6.1)\n",
            "Requirement already satisfied: ipython-genutils~=0.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets>5.0.0->sparkmagic) (0.2.0)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets>5.0.0->sparkmagic) (3.0.3)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from jedi>=0.10->ipython>=5->sparkmagic) (0.8.3)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.7/dist-packages (from jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (6.1.0)\n",
            "Collecting qtconsole\n",
            "  Downloading qtconsole-5.4.0-py3-none-any.whl (121 kB)\n",
            "\u001b[K     |████████████████████████████████| 121 kB 54.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (5.6.1)\n",
            "Requirement already satisfied: jinja2<=3.0.0 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (2.11.3)\n",
            "Requirement already satisfied: pyzmq>=17 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (23.2.1)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (0.13.3)\n",
            "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (0.15.0)\n",
            "Requirement already satisfied: jupyter-core>=4.4.0 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (4.11.2)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (1.8.0)\n",
            "Requirement already satisfied: nbformat in /usr/local/lib/python3.7/dist-packages (from notebook>=4.2->sparkmagic) (5.7.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2<=3.0.0->notebook>=4.2->sparkmagic) (2.0.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel>=4.2.2->sparkmagic) (2.8.2)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (0.8.4)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (0.4)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (5.0.1)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (1.5.0)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (0.6.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (0.7.1)\n",
            "Requirement already satisfied: importlib-metadata>=3.6 in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook>=4.2->sparkmagic) (4.13.0)\n",
            "Requirement already satisfied: fastjsonschema in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook>=4.2->sparkmagic) (2.16.2)\n",
            "Requirement already satisfied: jsonschema>=2.6 in /usr/local/lib/python3.7/dist-packages (from nbformat->notebook>=4.2->sparkmagic) (4.3.3)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=3.6->nbformat->notebook>=4.2->sparkmagic) (4.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=3.6->nbformat->notebook>=4.2->sparkmagic) (3.10.0)\n",
            "Requirement already satisfied: importlib-resources>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.2->sparkmagic) (5.10.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.2->sparkmagic) (0.18.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=2.6->nbformat->notebook>=4.2->sparkmagic) (22.1.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.17.1->sparkmagic) (2022.5)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from plotly>=3->autovizwidget>=0.6->sparkmagic) (8.1.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from plotly>=3->autovizwidget>=0.6->sparkmagic) (1.15.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython>=5->sparkmagic) (0.2.5)\n",
            "Collecting pyspnego[kerberos]\n",
            "  Downloading pyspnego-0.6.2-py3-none-any.whl (124 kB)\n",
            "\u001b[K     |████████████████████████████████| 124 kB 45.8 MB/s \n",
            "\u001b[?25hCollecting cryptography>=1.3\n",
            "  Downloading cryptography-38.0.3-cp36-abi3-manylinux_2_24_x86_64.whl (4.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.1 MB 44.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.7/dist-packages (from cryptography>=1.3->requests_kerberos>=0.8.0->sparkmagic) (1.15.1)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.12->cryptography>=1.3->requests_kerberos>=0.8.0->sparkmagic) (2.21)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->sparkmagic) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->sparkmagic) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->sparkmagic) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->sparkmagic) (1.24.3)\n",
            "Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook>=4.2->sparkmagic) (0.7.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (0.5.1)\n",
            "Collecting krb5>=0.3.0\n",
            "  Downloading krb5-0.4.1.tar.gz (218 kB)\n",
            "\u001b[K     |████████████████████████████████| 218 kB 45.3 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33m  WARNING: Generating metadata for package krb5 produced metadata for project name unknown. Fix your #egg=krb5 fragments.\u001b[0m\n",
            "\u001b[33mWARNING: Discarding https://files.pythonhosted.org/packages/56/f6/64874f52a47e652e37c8648665374e723500a62d82938ea2db842496d2ed/krb5-0.4.1.tar.gz#sha256=606059b9db52086f4e03e15b161d1d172ccc1172880e42fa5744f99a4407cf57 (from https://pypi.org/simple/krb5/) (requires-python:>=3.7). Requested unknown from https://files.pythonhosted.org/packages/56/f6/64874f52a47e652e37c8648665374e723500a62d82938ea2db842496d2ed/krb5-0.4.1.tar.gz#sha256=606059b9db52086f4e03e15b161d1d172ccc1172880e42fa5744f99a4407cf57 (from pyspnego[kerberos]->requests_kerberos>=0.8.0->sparkmagic) has inconsistent name: filename has 'krb5', but metadata has 'UNKNOWN'\u001b[0m\n",
            "  Downloading krb5-0.4.0.tar.gz (218 kB)\n",
            "\u001b[K     |████████████████████████████████| 218 kB 49.0 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33m  WARNING: Generating metadata for package krb5 produced metadata for project name unknown. Fix your #egg=krb5 fragments.\u001b[0m\n",
            "\u001b[33mWARNING: Discarding https://files.pythonhosted.org/packages/30/b2/fbafed4fe18a47bc3ffdd7a591bc02c4cf9e96d092793831cb98d596fa16/krb5-0.4.0.tar.gz#sha256=6eb0e91e044d37cb34c84b831310f4a48951f2902d4ebb65b97cbac5ff1b2264 (from https://pypi.org/simple/krb5/) (requires-python:>=3.7). Requested unknown from https://files.pythonhosted.org/packages/30/b2/fbafed4fe18a47bc3ffdd7a591bc02c4cf9e96d092793831cb98d596fa16/krb5-0.4.0.tar.gz#sha256=6eb0e91e044d37cb34c84b831310f4a48951f2902d4ebb65b97cbac5ff1b2264 (from pyspnego[kerberos]->requests_kerberos>=0.8.0->sparkmagic) has inconsistent name: filename has 'krb5', but metadata has 'UNKNOWN'\u001b[0m\n",
            "  Downloading krb5-0.3.0.tar.gz (1.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7 MB 55.7 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting gssapi>=1.6.0\n",
            "  Downloading gssapi-1.8.2.tar.gz (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 2.7 MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting qtpy>=2.0.1\n",
            "  Downloading QtPy-2.2.1-py3-none-any.whl (82 kB)\n",
            "\u001b[K     |████████████████████████████████| 82 kB 124 kB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from qtpy>=2.0.1->qtconsole->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (21.3)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->qtpy>=2.0.1->qtconsole->jupyter>=1->hdijupyterutils>=0.6->sparkmagic) (3.0.9)\n",
            "Building wheels for collected packages: sparkmagic, autovizwidget, hdijupyterutils, gssapi, krb5\n",
            "  Building wheel for sparkmagic (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sparkmagic: filename=sparkmagic-0.20.0-py3-none-any.whl size=66523 sha256=e45e7be3cd3bb64da3613219c748401a36e72c1d6914e060359820244d9bbf43\n",
            "  Stored in directory: /root/.cache/pip/wheels/3b/c8/8e/4f71d6a433ff3f41964583119c605d3d6bda4a43b4fd07c6e0\n",
            "  Building wheel for autovizwidget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for autovizwidget: filename=autovizwidget-0.20.0-py3-none-any.whl size=14642 sha256=0423cb0668a336b390e44e7056b64b504bb7d30356257505b8ade2c28394eff7\n",
            "  Stored in directory: /root/.cache/pip/wheels/a6/d1/e7/ff1252bfa2ff86f488c743b74a2e2e243e8e40aeb4ddeb1047\n",
            "  Building wheel for hdijupyterutils (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hdijupyterutils: filename=hdijupyterutils-0.20.0-py3-none-any.whl size=7660 sha256=1e94f2deae9a2821b5fb1ef5b5169fc2d7f123e52f38a3aba3e59953348d02a2\n",
            "  Stored in directory: /root/.cache/pip/wheels/ee/b9/c5/7b5cee27e18a2d7d380bc1538eead12e558ef6b60ea6870a99\n",
            "  Building wheel for gssapi (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gssapi: filename=gssapi-1.8.2-cp37-cp37m-linux_x86_64.whl size=2779397 sha256=8cbae12a9537262799a75378e4d48d3ee342a4150950311b3d99cd90f482ec59\n",
            "  Stored in directory: /root/.cache/pip/wheels/ea/19/45/5796e41bb54e7cf6334ca54e60a281b589a7e3318d05ba2d8a\n",
            "  Building wheel for krb5 (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for krb5: filename=krb5-0.3.0-cp37-cp37m-linux_x86_64.whl size=3073553 sha256=e6b317c24197379ef0da6a30f0bb477e679f671db986a83a0327de513b5378bd\n",
            "  Stored in directory: /root/.cache/pip/wheels/8d/9c/5b/c797df276a5e2d9fd6ffd544aade607a6edae4428c0b511093\n",
            "Successfully built sparkmagic autovizwidget hdijupyterutils gssapi krb5\n",
            "Installing collected packages: jedi, qtpy, qtconsole, cryptography, pyspnego, nose, mock, krb5, jupyter, gssapi, hdijupyterutils, requests-kerberos, nest-asyncio, autovizwidget, sparkmagic\n",
            "Successfully installed autovizwidget-0.20.0 cryptography-38.0.3 gssapi-1.8.2 hdijupyterutils-0.20.0 jedi-0.18.1 jupyter-1.0.0 krb5-0.3.0 mock-4.0.3 nest-asyncio-1.5.5 nose-1.3.7 pyspnego-0.6.2 qtconsole-5.4.0 qtpy-2.2.1 requests-kerberos-0.14.0 sparkmagic-0.20.0\n",
            "Looking in indexes: https://test.pypi.org/simple/, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting penn-grader==0.5.0\n",
            "  Downloading https://test-files.pythonhosted.org/packages/e4/fa/3a09f42fbed05d27fa4e663e3feed831bc88802342e1a862a6cfb09b77fc/penn_grader-0.5.0-py2.py3-none-any.whl (10 kB)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from penn-grader==0.5.0) (6.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from penn-grader==0.5.0) (0.3.6)\n",
            "Installing collected packages: penn-grader\n",
            "Successfully installed penn-grader-0.5.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6WAJmQ8IAbRs"
      },
      "source": [
        "%load_ext sparkmagic.magics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CL6n768EPt9E"
      },
      "source": [
        "### Step 0.2: The Sharp Spark\n",
        "\n",
        "Now, connect your notebook to the EMR cluster you created. In the first cell, copy the link to the Master Public DNS specified in the setup document. You will need to add `http://` to the beginning of the address and the auth details to the end.\n",
        "\n",
        "For example, if my DNS (directly from the AWS EMR console) is `ec2-3-15-237-211.us-east-2.compute.amazonaws.com` my address would be,\n",
        "\n",
        "`http://ec2-3-15-237-211.us-east-2.compute.amazonaws.com -a cis545-livy -p password1 -t Basic_Access`\n",
        "\n",
        "Insert this in the `# TODO # below`. For our example, the cell would read,\n",
        "\n",
        "```\n",
        "%spark add -s spark_session -l python -u http://ec2-3-15-237-211.us-east-2.compute.amazonaws.com -a cis545-livy -p password1 -t Basic_Access\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9QbylT-jqX9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 97,
          "referenced_widgets": [
            "53941c95cdf94f468aaabeace1e3e187",
            "509f4fe84c0d4d3098f2dc3069db3631",
            "a8d35f831b044f3d8152235c510a59b7"
          ]
        },
        "outputId": "5d599364-1a91-46a3-96ae-3f6df7412aec"
      },
      "source": [
        "# TODO: Copy the line above, enter your Master Public DNS with the proper formatting and host, and update the password\n",
        "\n",
        "%spark add -s spark_session -l python -u http://ec2-3-88-1-6.compute-1.amazonaws.com/ -a cis545-livy -p Johncena_999 -t Basic_Access"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Starting Spark application\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<table>\n",
              "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>User</th><th>Current session?</th></tr><tr><td>0</td><td>application_1667524272897_0001</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-16-24.ec2.internal:20888/proxy/application_1667524272897_0001/\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-26-42.ec2.internal:8042/node/containerlogs/container_1667524272897_0001_01_000001/livy\">Link</a></td><td>None</td><td>✔</td></tr></table>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "53941c95cdf94f468aaabeace1e3e187"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "SparkSession available as 'spark'.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QwKAHhQL0lf7"
      },
      "source": [
        "# If you ever need to restart, you may need to...\n",
        "# %spark delete -s my_session\n",
        "#OR just factory reset runtime under the runtime tab\n",
        "# %spark delete -s spark_session"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6yzJetbCfgh3"
      },
      "source": [
        "### Step 0.3: Cluster Log\n",
        "\n",
        "In order to keep track of clusters you have created and terminated as well as give us information about time spent on this assignment, please enter each date and time you created a cluster and the date and time you terminated the cluster. This will not impact your score."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "powXPhtphhyy"
      },
      "source": [
        "EX: \n",
        "\n",
        "10/12 9:00am - 10/12 12:00pm\n",
        "\n",
        "10/13 7:00pm - 10/13 9:00pm\n",
        "\n",
        "...\n",
        "    "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SokSZv4vih0c"
      },
      "source": [
        "TODO: Create cluster log"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#11/2 23:50-"
      ],
      "metadata": {
        "id": "Hh0uBL2onKnF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v1IQjUwNObb8"
      },
      "source": [
        "Enter your 8-digit Penn Key as an integer in the cell \n",
        "below. This will be used in the autograder.  **Please also update the cell below, with the same ID!**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7zM20juwqqQF"
      },
      "source": [
        "from penngrader.grader import *\n",
        "STUDENT_ID = 20723731"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile config.yaml\n",
        "\n",
        "grader_api_url: 'https://wyv616tp17.execute-api.us-east-1.amazonaws.com/default/Grader'\n",
        "grader_api_key: 'Kd32fl3g3p917iM0zwjiO23Bitj4PO9ga4LektOa'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ep52cRkLoBYE",
        "outputId": "eb2d64a3-be79-4ab6-f4c8-0183bc64ca3e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting config.yaml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V8Oo_5D7qoWp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8f3ac403-ae56-4150-9cb6-d99360b27d79"
      },
      "source": [
        "grader = PennGrader('config.yaml', 'CIS_5450_Fall22_HW3', STUDENT_ID, STUDENT_ID)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PennGrader initialized with Student ID: 20723731\n",
            "\n",
            "Make sure this correct or we will not be able to store your grade\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bfs-EQZUzF4j"
      },
      "source": [
        "Run the above cells to setup the autograder. Make sure to have set your 8 digit Penn ID in the cell above. It will also import all the modules you need for the homework.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Step 1: Data Wrangling, Cleaning, and Shaping [18 pts]\n",
        "\n",
        "In this homework we will be working with data from Amazon regarding their products and customer reviews. There are three JSON datasets with information about the products and three JSON datasets with information about the customer reviews for these products. The three categories of products are: Gourmet & Grocery Food, Toys & Games, and Home & Kitchen. \n",
        "\n",
        "\n",
        "The data you will use is stored in an S3 bucket, a cloud storage service. You now need to download it onto the nodes of your [EMR cluster](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-what-is-emr.html). \n",
        "\n",
        "\n",
        "*Citation for data:* \n",
        "\n",
        "- Image-based recommendations on styles and substitutes.\n",
        "J. McAuley, C. Targett, J. Shi, A. van den Hengel\n",
        "SIGIR, 2015\n",
        "\n",
        "- Ups and downs: Modeling the visual evolution of fashion trends with one-class collaborative filtering.\n",
        "R. He, J. McAuley\n",
        "WWW, 2016"
      ],
      "metadata": {
        "id": "4nyAa0wn1XCD"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nf_ADEXnIK0b"
      },
      "source": [
        "### Step 1.1: The Stupendous Schema\n",
        "\n",
        "When loading data, Spark will try to infer its structure on its own. This process is faulty because Spark will sometimes infer the type incorrectly. Spark's ability to determine types is not reliable, thus you will need to define a schema for [] and .\n",
        "\n",
        "A schema is a description of the structure of data. We have given you an example for the `products` datasets and you will be defining an explicit schema for the `reviews` datasets called ```schema_reviews```. \n",
        "\n",
        "\n",
        "In Spark, schemas are defined using a `StructType` object. This is a collection of data types, termed `StructField`'s, that specify the structure and variable type of each component of the dataset. For example, suppose we have the following simple JSON object,\n",
        "\n",
        "\n",
        "```\n",
        "{\n",
        " \"student_name\": \"Data Wrangler\",\n",
        " \"GPA\": 1.4,\n",
        " \"courses\": [\n",
        "    {\"department\": \"Computer and Information Science\",\n",
        "     \"course_id\": \"CIS 545\",\n",
        "     \"semester\": \"Fall 2021\"},\n",
        "    {\"department\": \"Computer and Information Science\",\n",
        "     \"course_id\": \"CIS 555\",\n",
        "     \"semester\": \"Fall 2021\"}\n",
        " ],\n",
        " \"grad_year\": 2022\n",
        " }\n",
        "```\n",
        "\n",
        "We would define its schema as follows,\n",
        "\n",
        "```       \n",
        "schema = StructType([\n",
        "           StructField(\"student_name\", StringType(), nullable=True),\n",
        "           StructField(\"GPA\", FloatType(), nullable=True),\n",
        "           StructField(\"courses\", ArrayType(\n",
        "                StructType([\n",
        "                  StructField(\"department\", StringType(), nullable=True),\n",
        "                  StructField(\"course_id\", StringType(), nullable=True),\n",
        "                  StructField(\"semester\", StringType(), nullable=True)\n",
        "                ])\n",
        "           ), nullable=True),\n",
        "           StructField(\"grad_year\", IntegerType(), nullable=True)\n",
        "         ])\n",
        "```\n",
        "\n",
        "\n",
        "Each `StructField` has the following structure: `(name, type, nullable)`. The `nullable` flag defines that the specified field may be empty. Your first task is to define the `schema` of the five `reviews` .json files. You can take a look at the schema for one of these files for reference. A smaller version of the `food_reviews` JSON dataset can be found here: https://drive.google.com/file/d/1mWYmGIhanT3fwB3cNkMSp67A6kkI_Mu3/view?usp=sharing\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "There is also no grading cell for this step.  But your JSON file won't load if it's wrong, so you have a way of testing.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pL-Ps4KWIJ9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "5f35dfe1662846abae4221a6a99e876a",
            "8b35960aca1f48faa096305703963fb7",
            "431cf91bef1e49fbb4185076fa878f2a"
          ]
        },
        "outputId": "2b1ea665-1115-4413-e26e-20e78f7fba7d"
      },
      "source": [
        "%%spark\n",
        "\n",
        "from pyspark.sql.types import *\n",
        "\n",
        "schema_products = StructType([\n",
        "          StructField(\"asin\", StringType(), nullable=True),\n",
        "           StructField(\"title\", StringType(), nullable=True),\n",
        "           StructField(\"price\", FloatType(), nullable=True),\n",
        "           StructField(\"imUrl\", StringType(), nullable=True),\n",
        "           StructField(\"related\", ArrayType(\n",
        "                StructType([\n",
        "                  StructField(\"also_bought\", ArrayType(StringType()), nullable=True),\n",
        "                  StructField(\"also_viewed\", ArrayType(StringType()), nullable=True),\n",
        "                  StructField(\"bought_together\", ArrayType(StringType()), nullable=True)\n",
        "                ])\n",
        "           ), nullable=True),\n",
        "           StructField(\"salesRank\", MapType(StringType(), StringType()), nullable=True),\n",
        "           StructField(\"brand\", StringType(), nullable=True),\n",
        "           StructField(\"categories\", ArrayType(StringType()), nullable=True),\n",
        "])\n",
        "\n",
        "#TODO: Create a schema for the reviews datasets\n",
        "schema_reviews = StructType([StructField(\"reviewerID\", StringType(), nullable=True),\n",
        "                             StructField(\"asin\", StringType(), nullable=True),\n",
        "                             StructField(\"reviewerName\", StringType(), nullable=True),\n",
        "                             StructField(\"helpful\", ArrayType(IntegerType()), nullable=True),\n",
        "                             StructField(\"reviewText\", StringType(), nullable=True),\n",
        "                             StructField(\"overall\", FloatType(), nullable=True),\n",
        "                             StructField(\"summary\", StringType(), nullable=True),\n",
        "                             StructField(\"unixReviewTime\", IntegerType(), nullable=True),\n",
        "                             StructField(\"reviewTime\", StringType(), nullable=True)])\n",
        "\n",
        "\n",
        "\n",
        "          \n",
        "          "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5f35dfe1662846abae4221a6a99e876a"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Su604X9ggc2"
      },
      "source": [
        "### Step 1.2: The Langorous Load [9 pts]\n",
        "\n",
        "In the following cell, load the three `reviews` datasets and the three `products` datsets from your S3 bucket into Spark dataframes (sdf). If you have constructed `schema_reviews` and `schema_products` correctly, `spark.read.json()` will read in the datasets. The names of the datasets are listed below:\n",
        "\n",
        "Reviews:\n",
        "\n",
        "- `food_reviews.json`\n",
        "- `home_kitchen_reviews.json`\n",
        "- `toys_games_reviews.json`\n",
        "\n",
        "Products:\n",
        "\n",
        "- `food_metadata.json`\n",
        "- `home_kitchen_meta.json`\n",
        "- `toys_games_meta.json`\n",
        "\n",
        "\n",
        "\n",
        "***You do not need to edit this cell***. If this doesn't work, go back to the prior cell and update your schema. Note that the cell below will load data even if your schema is incomplete and has left out some columns of the data, so be sure to check that you have included all of the fields from the JSON."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "food_reviews_sdf = spark.read.json(\"s3://penn-cis545-files/food_reviews.json\", schema=schema_reviews, multiLine = False, primitivesAsString = True)\n",
        "home_kitchen_reviews_sdf = spark.read.json(\"s3://penn-cis545-files/home_kitchen_reviews.json\", schema=schema_reviews, multiLine = False, primitivesAsString = True)\n",
        "toys_games_reviews_sdf = spark.read.json(\"s3://penn-cis545-files/toys_games_reviews.json\", schema=schema_reviews, multiLine = False, primitivesAsString = True)\n"
      ],
      "metadata": {
        "id": "Tcpx-SExfyJa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "7c974b5a06b14b1cb81c32706ee88cd4",
            "c888352c732d45619eeb3a99b98a02fe",
            "50b7c0fd9d36464ca8f4cd6f780592f3"
          ]
        },
        "outputId": "b916eb24-9d6a-4e1a-ba10-d932e28c336f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7c974b5a06b14b1cb81c32706ee88cd4"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "# Let's print out the first few rows to see how the data looks like in tabular form\n",
        "home_kitchen_reviews_sdf.show(5)"
      ],
      "metadata": {
        "id": "eAgWjMU3ikD_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196,
          "referenced_widgets": [
            "e1f62c4d1df241c7ad24fe22ef01ff55",
            "bf932f6db5324206b7eb565a4e1baef4",
            "c112235b3f4f45ff93f8fce293526e6f"
          ]
        },
        "outputId": "f437e8fa-86f8-4f8d-d53d-c63ed0f0c8b0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e1f62c4d1df241c7ad24fe22ef01ff55"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+----------+-----------------+--------+--------------------+-------+--------------------+--------------+-----------+\n",
            "|    reviewerID|      asin|     reviewerName| helpful|          reviewText|overall|             summary|unixReviewTime| reviewTime|\n",
            "+--------------+----------+-----------------+--------+--------------------+-------+--------------------+--------------+-----------+\n",
            "|A210NOCSTBT4OD|0076144011|           Sheila|  [0, 0]|Have you ever tho...|    4.0|              Lovely|    1349308800| 10 4, 2012|\n",
            "|A28ILV4TOG8BH2|0130350591|         ccjensen|  [0, 0]|The butter dish i...|    5.0|Nice looking, and...|    1300752000|03 22, 2011|\n",
            "|A31B4D7URW4DNZ|0307394530|           3Gigi3|[11, 16]|I anxiously waite...|    2.0| Mother of the Bride|    1214784000|06 30, 2008|\n",
            "|A2HU0RPDRZZOP1|0307394530|   Alexey Leontev|  [0, 0]|Fantastic book, a...|    5.0|        amazing book|    1277337600|06 24, 2010|\n",
            "| A7J0XOW7DYBBD|0307394530|Allan Mar Cariaso|  [0, 0]|Can't wait to try...|    5.0|        Very helpful|    1393113600|02 23, 2014|\n",
            "+--------------+----------+-----------------+--------+--------------------+-------+--------------------+--------------+-----------+\n",
            "only showing top 5 rows"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create a spark dataframe for food_prod_sdf, home_kitchen_prod_sdf, and toys_games_prod_sdf\n",
        "food_prod_sdf = spark.read.json(\"s3://penn-cis545-files/food_metadata.json\", schema=schema_products, multiLine=False, primitivesAsString=True)\n",
        "home_kitchen_prod_sdf = spark.read.json(\"s3://penn-cis545-files/home_kitchen_meta.json\", schema=schema_products, multiLine=False, primitivesAsString=True)\n",
        "toys_games_prod_sdf = spark.read.json(\"s3://penn-cis545-files/toys_games_meta.json\", schema=schema_products, multiLine=False, primitivesAsString=True)\n"
      ],
      "metadata": {
        "id": "ff6o83l3f-Af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "a1340938cc2c4a3db3fcb3527479cc8d",
            "ff2570d76eea4a8fadece45387460ca0",
            "9088a788aef742269e25a6780c2e1847"
          ]
        },
        "outputId": "db908a57-147d-481b-ebf4-e84f80fe2fc3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a1340938cc2c4a3db3fcb3527479cc8d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMVCVotcE1wv"
      },
      "source": [
        "The cell below shows how to run SQL commands on Spark tables. Use this as a template for all your SQL queries in this notebook. \n",
        "\n",
        "***You do not need to edit this cell***."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "EJ0aRJM-NEYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "# Create SQL-accesible table\n",
        "food_reviews_sdf.createOrReplaceTempView(\"food_reviews\")\n",
        "\n",
        "# Declare SQL query to be excecuted\n",
        "query = '''SELECT * \n",
        "           FROM food_reviews ORDER BY `asin` DESC LIMIT 10'''\n",
        "\n",
        "# Save the output sdf of spark.sql() as answer_review_sdf and convert to Pandas\n",
        "answer_review_sdf = spark.sql(query)\n",
        "answer_review_sdf.show()\n"
      ],
      "metadata": {
        "id": "7CnbKKqtnicU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "c3abe6dfac684b8aad2532fdbf9457ef",
            "1d3bfb554be440108560cea488dace9f",
            "472e72e23fa04378882464407dea402d"
          ]
        },
        "outputId": "75db9e22-e4b0-4d95-f030-5682e5761a16"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c3abe6dfac684b8aad2532fdbf9457ef"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+----------+--------------------+-------+--------------------+-------+--------------------+--------------+-----------+\n",
            "|    reviewerID|      asin|        reviewerName|helpful|          reviewText|overall|             summary|unixReviewTime| reviewTime|\n",
            "+--------------+----------+--------------------+-------+--------------------+-------+--------------------+--------------+-----------+\n",
            "|A2NSZZ7Y0RAE45|B00LQWKDBM|Michael Freed \"fr...| [0, 0]|Makes AMAZING mea...|    5.0|    AMAZING meatloaf|    1405123200|07 12, 2014|\n",
            "|A1DGBQQIBHTLPF|B00LOZ7F0S|     Berkley Roberts| [0, 0]|I was attracted t...|    5.0|Great Appeal and ...|    1405555200|07 17, 2014|\n",
            "|A3DYJ90MXMH04P|B00LOZ7F0S|    Maximiliano Korp| [1, 1]|Wonderful vanilla...|    5.0|Wonderful vanilla...|    1404950400|07 10, 2014|\n",
            "| AWQYAJV2QWLWA|B00LOZ7F0S|      Micheal Taylor| [0, 0]|I usually don't w...|    5.0|The best I have f...|    1405209600|07 13, 2014|\n",
            "|A2AT0AI6QKSY2C|B00LOZ7F0S|          Willy Korp| [0, 0]|I like to drink m...|    5.0|     Healthy product|    1405382400|07 15, 2014|\n",
            "| A6B61KJ12Z9SV|B00LMMLRG6| cynister \"cynister\"| [1, 1]|Cheesy, Crunchy, ...|    5.0|Cheesy, Crunchy, ...|    1405382400|07 15, 2014|\n",
            "| AMJB5BFAUE081|B00LDXFI6Y|          Richard C.| [0, 0]|Great product. I ...|    4.0|Great with conden...|    1405728000|07 19, 2014|\n",
            "|A2DG81T8YR0PTM|B00LDOM4OM|               Joycy| [0, 0]|If you like peanu...|    5.0|           Delicious|    1404086400|06 30, 2014|\n",
            "| ACF0NFD95VRUS|B00LDOM4OM|      Kris V. Warfel| [0, 0]|It is spicy but n...|    5.0|          good stuff|    1404086400|06 30, 2014|\n",
            "|A12XIXE9S64TF7|B00LDBMZ9O|xtina \"moy drook ...| [0, 0]|I cannot recommen...|    5.0|I cannot recommen...|    1404950400|07 10, 2014|\n",
            "+--------------+----------+--------------------+-------+--------------------+-------+--------------------+--------------+-----------+"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MLF-swRBTuZG"
      },
      "source": [
        "We can then copy the `answer_review_sdf` to Colab to submit to PennGrader..."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o answer_review_sdf"
      ],
      "metadata": {
        "id": "2z-mDBTorcX6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "6da280f9b05248a8b677e49757559957",
            "7b0dcb00efdb4e4db15759d4a9c8c241",
            "c4d54790c85d443393fd6ffb26e512d2",
            "0ea241ae625b465b8bbb94ca7b755469",
            "09ddd01e844941fa825abc8249b7bf43",
            "ad7be9f0560d4ce1a805d392c3f0fc73"
          ]
        },
        "outputId": "2437ce2a-ce9a-401f-9545-349845ca0dd7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6da280f9b05248a8b677e49757559957"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0ea241ae625b465b8bbb94ca7b755469"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "answer_review_sdf"
      ],
      "metadata": {
        "id": "GeB6VJVSsGmk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "outputId": "583394a0-9a31-4a3c-f7d2-98cc0ede10a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       reviewerID        asin             reviewerName helpful  \\\n",
              "0  A2NSZZ7Y0RAE45  B00LQWKDBM  Michael Freed \"freedml\"  [0, 0]   \n",
              "1  A1DGBQQIBHTLPF  B00LOZ7F0S          Berkley Roberts  [0, 0]   \n",
              "2  A3DYJ90MXMH04P  B00LOZ7F0S         Maximiliano Korp  [1, 1]   \n",
              "3   AWQYAJV2QWLWA  B00LOZ7F0S           Micheal Taylor  [0, 0]   \n",
              "4  A2AT0AI6QKSY2C  B00LOZ7F0S               Willy Korp  [0, 0]   \n",
              "5   A6B61KJ12Z9SV  B00LMMLRG6      cynister \"cynister\"  [1, 1]   \n",
              "6   AMJB5BFAUE081  B00LDXFI6Y               Richard C.  [0, 0]   \n",
              "7  A2DG81T8YR0PTM  B00LDOM4OM                    Joycy  [0, 0]   \n",
              "8   ACF0NFD95VRUS  B00LDOM4OM           Kris V. Warfel  [0, 0]   \n",
              "9  A12XIXE9S64TF7  B00LDBMZ9O  xtina \"moy drook reads\"  [0, 0]   \n",
              "\n",
              "                                          reviewText  overall  \\\n",
              "0  Makes AMAZING meatloaf.  It's our &#34;old fam...      5.0   \n",
              "1  I was attracted to this product initially beca...      5.0   \n",
              "2  Wonderful vanilla. Very strong vanilla flavor,...      5.0   \n",
              "3  I usually don't write reviews but this product...      5.0   \n",
              "4  I like to drink my tea with vanilla extract, i...      5.0   \n",
              "5  Cheesy, Crunchy, Salty goodness. Exactly what ...      5.0   \n",
              "6  Great product. I usually add some sweetened co...      4.0   \n",
              "7  If you like peanuts and wasabi peas, you'll lo...      5.0   \n",
              "8  It is spicy but not overpowering and thes ship...      5.0   \n",
              "9  I cannot recommend these highly enough. I thin...      5.0   \n",
              "\n",
              "                                         summary  unixReviewTime reviewTime  \n",
              "0                               AMAZING meatloaf      1405123200 2014-07-12  \n",
              "1                      Great Appeal and Quality!      1405555200 2014-07-17  \n",
              "2  Wonderful vanilla. Very strong vanilla flavor      1404950400 2014-07-10  \n",
              "3                          The best I have found      1405209600 2014-07-13  \n",
              "4                                Healthy product      1405382400 2014-07-15  \n",
              "5               Cheesy, Crunchy, Salty goodness.      1405382400 2014-07-15  \n",
              "6                      Great with condensed milk      1405728000 2014-07-19  \n",
              "7                                      Delicious      1404086400 2014-06-30  \n",
              "8                                     good stuff      1404086400 2014-06-30  \n",
              "9         I cannot recommend these highly enough      1404950400 2014-07-10  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c2f50514-4e2a-4bb0-b1d0-adb6b1e005c6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviewerID</th>\n",
              "      <th>asin</th>\n",
              "      <th>reviewerName</th>\n",
              "      <th>helpful</th>\n",
              "      <th>reviewText</th>\n",
              "      <th>overall</th>\n",
              "      <th>summary</th>\n",
              "      <th>unixReviewTime</th>\n",
              "      <th>reviewTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>A2NSZZ7Y0RAE45</td>\n",
              "      <td>B00LQWKDBM</td>\n",
              "      <td>Michael Freed \"freedml\"</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>Makes AMAZING meatloaf.  It's our &amp;#34;old fam...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>AMAZING meatloaf</td>\n",
              "      <td>1405123200</td>\n",
              "      <td>2014-07-12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A1DGBQQIBHTLPF</td>\n",
              "      <td>B00LOZ7F0S</td>\n",
              "      <td>Berkley Roberts</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>I was attracted to this product initially beca...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Great Appeal and Quality!</td>\n",
              "      <td>1405555200</td>\n",
              "      <td>2014-07-17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>A3DYJ90MXMH04P</td>\n",
              "      <td>B00LOZ7F0S</td>\n",
              "      <td>Maximiliano Korp</td>\n",
              "      <td>[1, 1]</td>\n",
              "      <td>Wonderful vanilla. Very strong vanilla flavor,...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Wonderful vanilla. Very strong vanilla flavor</td>\n",
              "      <td>1404950400</td>\n",
              "      <td>2014-07-10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AWQYAJV2QWLWA</td>\n",
              "      <td>B00LOZ7F0S</td>\n",
              "      <td>Micheal Taylor</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>I usually don't write reviews but this product...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>The best I have found</td>\n",
              "      <td>1405209600</td>\n",
              "      <td>2014-07-13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A2AT0AI6QKSY2C</td>\n",
              "      <td>B00LOZ7F0S</td>\n",
              "      <td>Willy Korp</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>I like to drink my tea with vanilla extract, i...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Healthy product</td>\n",
              "      <td>1405382400</td>\n",
              "      <td>2014-07-15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>A6B61KJ12Z9SV</td>\n",
              "      <td>B00LMMLRG6</td>\n",
              "      <td>cynister \"cynister\"</td>\n",
              "      <td>[1, 1]</td>\n",
              "      <td>Cheesy, Crunchy, Salty goodness. Exactly what ...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Cheesy, Crunchy, Salty goodness.</td>\n",
              "      <td>1405382400</td>\n",
              "      <td>2014-07-15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AMJB5BFAUE081</td>\n",
              "      <td>B00LDXFI6Y</td>\n",
              "      <td>Richard C.</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>Great product. I usually add some sweetened co...</td>\n",
              "      <td>4.0</td>\n",
              "      <td>Great with condensed milk</td>\n",
              "      <td>1405728000</td>\n",
              "      <td>2014-07-19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>A2DG81T8YR0PTM</td>\n",
              "      <td>B00LDOM4OM</td>\n",
              "      <td>Joycy</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>If you like peanuts and wasabi peas, you'll lo...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Delicious</td>\n",
              "      <td>1404086400</td>\n",
              "      <td>2014-06-30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ACF0NFD95VRUS</td>\n",
              "      <td>B00LDOM4OM</td>\n",
              "      <td>Kris V. Warfel</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>It is spicy but not overpowering and thes ship...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>good stuff</td>\n",
              "      <td>1404086400</td>\n",
              "      <td>2014-06-30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>A12XIXE9S64TF7</td>\n",
              "      <td>B00LDBMZ9O</td>\n",
              "      <td>xtina \"moy drook reads\"</td>\n",
              "      <td>[0, 0]</td>\n",
              "      <td>I cannot recommend these highly enough. I thin...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>I cannot recommend these highly enough</td>\n",
              "      <td>1404950400</td>\n",
              "      <td>2014-07-10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c2f50514-4e2a-4bb0-b1d0-adb6b1e005c6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c2f50514-4e2a-4bb0-b1d0-adb6b1e005c6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c2f50514-4e2a-4bb0-b1d0-adb6b1e005c6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, you will implement the exact same thing yourself for `food_prod_sdf` and submit it to the grader. Select all columns from `food_prod` and order by the `asin` column (descending). Limit your output to only 10 rows and save your output dataframe to Colab as answer_prod_sdf. "
      ],
      "metadata": {
        "id": "-A9nzeVaG4QV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "# Create SQL-accesible table\n",
        "food_prod_sdf.createOrReplaceTempView(\"food_prod\")\n",
        "\n",
        "# Declare SQL query to be excecuted\n",
        "query=\"\"\"\n",
        "SELECT * \n",
        "FROM food_prod\n",
        "ORDER BY asin DESC\n",
        "LIMIT 10\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "# Save the output sdf of spark.sql() as answer_prod_sdf \n",
        "answer_prod_sdf=spark.sql(query)\n",
        "answer_prod_sdf.show()"
      ],
      "metadata": {
        "id": "9ZGUCgYCoVxY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "93f5cfb8aec64cd09de0383d6c9cb32e",
            "42d26ad24a7940978850ea4d34a22f92",
            "0c88f1a77ad140728781e85d7d5a727b"
          ]
        },
        "outputId": "896f6f8c-ced9-42d1-fab0-b7211b1d3efa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "93f5cfb8aec64cd09de0383d6c9cb32e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+------+--------------------+-------+--------------------+-----+--------------------+\n",
            "|      asin|               title| price|               imUrl|related|           salesRank|brand|          categories|\n",
            "+----------+--------------------+------+--------------------+-------+--------------------+-----+--------------------+\n",
            "|B00LQWKDBM|Mrs. Grass Onion ...|  null|http://ecx.images...|   null|                null| null|[[\"Grocery & Gour...|\n",
            "|B00LOZ7F0S|Organic Mexican V...|  null|http://ecx.images...|   null|                null| null|[[\"Grocery & Gour...|\n",
            "|B00LOXAZ1Q|Sour Punch Blue R...| 16.55|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|\n",
            "|B00LMMLRG6|Moon Cheese, 2 Oz...| 16.95|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|\n",
            "|B00LDXFI6Y|Nescafe Cafe Viet...| 17.99|http://ecx.images...|   null|                null| null|[[\"Grocery & Gour...|\n",
            "|B00LDOM4OM|Jessy Home Green ...|  null|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|\n",
            "|B00LDBMZ9O|Bark Thins Snacki...| 38.99|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|\n",
            "|B00LD1HC3I|Nut Bites (Gluten...|  null|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|\n",
            "|B00LC9PSI2|Garden of Life Ra...|119.99|http://ecx.images...|   null|{Health & Persona...| null|[[\"Grocery & Gour...|\n",
            "|B00LB86WQ6|Ballard Extracts,...|  40.0|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|\n",
            "+----------+--------------------+------+--------------------+-------+--------------------+-----+--------------------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o answer_prod_sdf"
      ],
      "metadata": {
        "id": "-_-HI3SWO2nh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "01736961e1474b3ba131162f242a2664",
            "470944545c534a2f85b4fca11efe6568",
            "0dea508687d14f3d984fa0f3e7244aab",
            "5ea868974d914e4295dbad2b192168f3",
            "dc32fa60b62e452b94034d71f723fe51",
            "135a21cb249843eb92eab7e8074691fa"
          ]
        },
        "outputId": "aa8cd6a1-d5b9-40a1-c87a-8659186b2b5f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "01736961e1474b3ba131162f242a2664"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5ea868974d914e4295dbad2b192168f3"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## AUTOGRADER \n",
        "#4 points\n",
        "grader.grade(test_case_id = 'test_answer_prod', answer = answer_prod_sdf)"
      ],
      "metadata": {
        "id": "z5T4kv0jpPLU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "67f3b093-e8dc-41ff-f6da-b6b6f089fdbe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 4/4 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the next cell, we want to find the ten most expensive food products. Create `food_prices_sdf` to fetch the data from `food_prod_sdf` table, returning rows with schema `(asin,title,price)` and ordered by price (descending). Limit your sdf to 10 rows. Save your final answer to Colab to submit to PennGrader.\n"
      ],
      "metadata": {
        "id": "G81VMofArbSD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark \n",
        "\n",
        "# TODO: create food_prices_sdf\n",
        "query =\"\"\"\n",
        "SELECT asin, title, price\n",
        "FROM food_prod\n",
        "ORDER BY price DESC\n",
        "LIMIT 10\n",
        "\"\"\"\n",
        "\n",
        "food_prices_sdf=spark.sql(query)\n",
        "food_prices_sdf.show()"
      ],
      "metadata": {
        "id": "FiX4gS6-sgal",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "6cb675ba10ff46929fcda37cfea8c1c5",
            "51f9861db1b54458ad6ca8dfe28c47eb",
            "11ea21142ac84ba4ad174635856cdab1"
          ]
        },
        "outputId": "5523d4c1-f56c-467f-f5d3-73de3ddb6592"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6cb675ba10ff46929fcda37cfea8c1c5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+------+\n",
            "|      asin|               title| price|\n",
            "+----------+--------------------+------+\n",
            "|B007RAMAA4|Havista Dried Shi...|999.99|\n",
            "|B005GX8N7S|Barbara's Bakery ...|999.52|\n",
            "|B006TN0I5O|Italian Pizza Sou...| 960.0|\n",
            "|B0029XZAUQ|Sharp, Provolone ...|857.26|\n",
            "|B001J87TOW|BOSS BAR ADVANCE ...| 854.8|\n",
            "|B001E5DC0U|Taste Specialty F...|835.45|\n",
            "|B002EPJQCM|Marcona Almonds -...|818.15|\n",
            "|B002AQPVEC|Jamon Iberico de ...| 799.0|\n",
            "|B004TABDRG|FLAVIA Creation 4...| 799.0|\n",
            "|B00DORGCGM|2012 Kongming Gon...| 789.0|\n",
            "+----------+--------------------+------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o food_prices_sdf"
      ],
      "metadata": {
        "id": "5eXGu4RTRD-k",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "0075b5ac83fb4f119f593f6c8470e9e5",
            "6f41d88f46874e3d814335e7453f1b1e",
            "7755c1b7f04d44bbba47650676fe1ab1",
            "089bd7251f1a430380d1b3f58382c01b",
            "b3c995aa2fdf4d759239d1bd7f74b215",
            "4a784f0c852d4dd0aaf7cdc92cab1fa6"
          ]
        },
        "outputId": "01d3730c-9837-43e0-f286-f6ce244070ba"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0075b5ac83fb4f119f593f6c8470e9e5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "089bd7251f1a430380d1b3f58382c01b"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 5 points\n",
        "grader.grade(test_case_id = 'test_food_price', answer = food_prices_sdf)"
      ],
      "metadata": {
        "id": "LEDWWnGTs25a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c5a7ca78-a4ae-4d41-b974-de8e49322565"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 5/5 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 1.3: Further Cleaning [9 pts]\n",
        "\n",
        "Before we move on to the later parts of this homework, we want to do some cleaning of our datasets. Currently, our datasets are divided based on their category (Food is in its own table, Home and Kitchen is its own table, etc.), but we eventually want to combine our dataframes so that they are easier to work with. If we do this, we want to be able to discern what category each product belongs to, but there currently does not exist any column that tells us the category of a product.\n",
        "\n",
        "\n",
        "For `food_prod_sdf`, create a new column called `category` where every value in this column will be 'Food'. Name this dataframe as `food_prod_final_sdf`. Repeat this for `toys_games_prod_sdf` and `home_kitchen_prod_sdf.` In `toys_games_prod_sdf`, every value in the `category` column will be 'Toys & Games'. In `home_kitchen_prod_sdf`, every value in this column will be 'Home & Kitchen'. Name these dataframes as `toys_games_prod_final_sdf` and `home_kitchen_prod_final_sdf`\n"
      ],
      "metadata": {
        "id": "-2vq-3ZfBYZD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create food_prod_final_sdf\n",
        "\n",
        "query =\"\"\"\n",
        "SELECT *,\"Food\" AS category\n",
        "FROM food_prod\n",
        "\"\"\"\n",
        "\n",
        "food_prod_final_sdf=spark.sql(query)\n",
        "food_prod_final_sdf.show(5)\n"
      ],
      "metadata": {
        "id": "ReMK5P097cZo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196,
          "referenced_widgets": [
            "6820f35f0403466a99813587a84fdfe8",
            "b80833acd2d04441819e2d1b8971291f",
            "5e5b26e4911c42d78614c1472b122b20"
          ]
        },
        "outputId": "21d66fbc-a044-4942-aab9-b78929f287ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6820f35f0403466a99813587a84fdfe8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+-----+--------------------+-------+--------------------+-----+--------------------+--------+\n",
            "|      asin|               title|price|               imUrl|related|           salesRank|brand|          categories|category|\n",
            "+----------+--------------------+-----+--------------------+-------+--------------------+-----+--------------------+--------+\n",
            "|      null|                null| null|                null|   null|                null| null|                null|    Food|\n",
            "|0700026444|Pure Darjeeling T...| null|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|    Food|\n",
            "|1403796890|WWE Kids Todler V...| 3.99|http://ecx.images...|   null|                null| null|[[\"Grocery & Gour...|    Food|\n",
            "|141278509X|Archer Farms Stra...| null|http://ecx.images...|   null|{Grocery & Gourme...| null|[[\"Grocery & Gour...|    Food|\n",
            "|1453060375|Mio Energy Liquid...|11.99|http://ecx.images...|   null|{Grocery & Gourme...|  Mio|[[\"Grocery & Gour...|    Food|\n",
            "+----------+--------------------+-----+--------------------+-------+--------------------+-----+--------------------+--------+\n",
            "only showing top 5 rows"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "# Do not edit\n",
        "answer_food_final_sdf = food_prod_final_sdf.alias('answer_food_final_sdf')"
      ],
      "metadata": {
        "id": "R1xgTOgRkbTo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "3cc898c09eca4a2dbb75eff34c03668d",
            "b92ea60ea6a848bc862ce1f544da5e0e",
            "4799c55f47d64ce5a1e5c2f157592103"
          ]
        },
        "outputId": "21c1e511-652a-4331-dec3-b1711d5e20cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3cc898c09eca4a2dbb75eff34c03668d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o answer_food_final_sdf"
      ],
      "metadata": {
        "id": "kY3tkb5UBIFV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "ad0e3b506276483ea11aaab9d998f9ea",
            "9308e999af06414ea6d9a861157992d5",
            "3537a8712ade4c81bb2cbbb7f523ac91",
            "b2ff07681ace472899bee8aff4d6038f",
            "91c8452eaa134ac09460c6b4bcc977a7",
            "0f04f931589f4cd5a04d4905deaef2cb"
          ]
        },
        "outputId": "fbf1e266-3b08-402c-da38-0eef4c40e59f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ad0e3b506276483ea11aaab9d998f9ea"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b2ff07681ace472899bee8aff4d6038f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2 points\n",
        "grader.grade(test_case_id = 'test_food_prod_final', answer = answer_food_final_sdf)"
      ],
      "metadata": {
        "id": "hHwISpDBBKDH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e6f9c1a7-3397-42ff-f63a-2684c64e6587"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 2/2 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create home_kitchen_prod_final_sdf\n",
        "home_kitchen_prod_sdf.createOrReplaceTempView(\"home_kitchen\")\n",
        "query =\"\"\"\n",
        "SELECT *,\"Home & Kitchen\" AS category\n",
        "FROM home_kitchen\n",
        "\"\"\"\n",
        "\n",
        "home_kitchen_prod_final_sdf=spark.sql(query)\n",
        "home_kitchen_prod_final_sdf.show(5)"
      ],
      "metadata": {
        "id": "VC6ixe149g2x",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196,
          "referenced_widgets": [
            "706102b434e14e62b34668503972784b",
            "2000ac6d2ded45aebc86b000f40810ba",
            "b742074707ff443380530b6786c77a6f"
          ]
        },
        "outputId": "33cd0c11-98a7-4638-9f91-66e0ebcab606"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "706102b434e14e62b34668503972784b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+-----+--------------------+-------+--------------------+------------+--------------------+--------------+\n",
            "|      asin|               title|price|               imUrl|related|           salesRank|       brand|          categories|      category|\n",
            "+----------+--------------------+-----+--------------------+-------+--------------------+------------+--------------------+--------------+\n",
            "|0076144011|Ninjas, Piranhas,...| null|http://g-ecx.imag...|   null|  {Books -> 6285595}|        null|[[\"Home & Kitchen\"]]|Home & Kitchen|\n",
            "|0130350591|Le Creuset Kiwi (...| null|http://ecx.images...|   null|{Kitchen & Dining...|        null|[[\"Home & Kitchen...|Home & Kitchen|\n",
            "|0307394530|Martha Stewart's ...|14.99|http://ecx.images...|   null|{Arts, Crafts & S...|Random House|[[\"Home & Kitchen...|Home & Kitchen|\n",
            "|0439903491|                null|29.99|http://ecx.images...|   null|  {Software -> 7065}|        null|[[\"Home & Kitchen...|Home & Kitchen|\n",
            "|0578060604|Build A Maloof In...| null|http://ecx.images...|   null|                null|        null|[[\"Home & Kitchen...|Home & Kitchen|\n",
            "+----------+--------------------+-----+--------------------+-------+--------------------+------------+--------------------+--------------+\n",
            "only showing top 5 rows"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "# Do not edit\n",
        "answer_home_kitchen_sdf = home_kitchen_prod_final_sdf.alias('answer_home_kitchen_sdf')"
      ],
      "metadata": {
        "id": "4w8ReS5Im_Q1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "cd502149295243cc81f65c7988a02331",
            "bbbd324f47674fe0ae49f103696de8cd",
            "69d0aeefbdbe4c33950bcfd848a5852e"
          ]
        },
        "outputId": "95701121-94b5-451e-f21f-eff1481bd291"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cd502149295243cc81f65c7988a02331"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o answer_home_kitchen_sdf"
      ],
      "metadata": {
        "id": "LigSdgcfBVz2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "323a668fa3d04097b7b90fcd1f575705",
            "0afab056c19847f38ec5687d887a5a47",
            "9ac634d36ec74968b15bae732a643b97",
            "bd880327728749eeba445d1f819a1e04",
            "9a95e5cb8ef549838254729e399b5f9d",
            "7f22f00142fe40a78a69c5942bebbf1d"
          ]
        },
        "outputId": "aaa2534c-1f46-4b04-bae8-6cd75d1e88b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "323a668fa3d04097b7b90fcd1f575705"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd880327728749eeba445d1f819a1e04"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2 points\n",
        "grader.grade(test_case_id = 'test_home_kitchen_final', answer = answer_home_kitchen_sdf)"
      ],
      "metadata": {
        "id": "vsouyFm-BX37",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "446b3504-d319-44aa-8b8b-bc22b4ecb4ac"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 2/2 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create toys_games_prod_final_sdf\n",
        "toys_games_prod_sdf.createOrReplaceTempView(\"toys_games\")\n",
        "query =\"\"\"\n",
        "SELECT *,\"Toys & Games\" AS category\n",
        "FROM toys_games\n",
        "\"\"\"\n",
        "\n",
        "toys_games_prod_final_sdf=spark.sql(query)\n",
        "toys_games_prod_final_sdf.show(5)"
      ],
      "metadata": {
        "id": "gs4yaHLY9s3R",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196,
          "referenced_widgets": [
            "9e6cb707b4a9443e8bfe09ff8aecb401",
            "f855fd4dd9a74e7ca958fb8ea6b4cf0b",
            "6d19c25209ab494983c9442552276d55"
          ]
        },
        "outputId": "ca29d018-03ad-4dbe-b30f-ed0be68b437b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9e6cb707b4a9443e8bfe09ff8aecb401"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+------+--------------------+-------+--------------------+-------------+--------------------+------------+\n",
            "|      asin|               title| price|               imUrl|related|           salesRank|        brand|          categories|    category|\n",
            "+----------+--------------------+------+--------------------+-------+--------------------+-------------+--------------------+------------+\n",
            "|0000191639|Dr. Suess 19163 D...| 37.12|http://ecx.images...|   null|{Toys & Games -> ...|    Dr. Seuss|[[\"Toys & Games\",...|Toys & Games|\n",
            "|0005069491|Nursery Rhymes Fe...|  null|http://ecx.images...|   null|{Toys & Games -> ...|         null|  [[\"Toys & Games\"]]|Toys & Games|\n",
            "|0076561046|Fraction Decimal ...|  null|http://ecx.images...|   null|{Toys & Games -> ...|         null|[[\"Toys & Games\",...|Toys & Games|\n",
            "|0131358936|                null| 36.22|http://ecx.images...|   null|  {Software -> 8080}|         null|[[\"Toys & Games\",...|Toys & Games|\n",
            "|0133642984|Algebra 2 Califor...|731.93|http://ecx.images...|   null|{Toys & Games -> ...|Prentice Hall|[[\"Toys & Games\",...|Toys & Games|\n",
            "+----------+--------------------+------+--------------------+-------+--------------------+-------------+--------------------+------------+\n",
            "only showing top 5 rows"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "# Do not edit\n",
        "answer_toys_games_sdf = toys_games_prod_final_sdf.alias('answer_toys_games_sdf')"
      ],
      "metadata": {
        "id": "dDQiupo1nKoI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "0d6c2f1430ef4382896116fe4eacbb26",
            "9dd7aa464b584daa81a8889281f0c3bd",
            "c2d0fc4b55c04e50baace1715d0b8476"
          ]
        },
        "outputId": "0101a05e-fcde-44c2-9755-e6d1831f7f57"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0d6c2f1430ef4382896116fe4eacbb26"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o answer_toys_games_sdf"
      ],
      "metadata": {
        "id": "SpyBoucbBaEx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "734e83870e8845d696fcbe8132a375f5",
            "23124827d61141139e4f5f37f72a01b0",
            "e589d85e64564bc096829ff49354d911",
            "976b8a287dc14898a17236f13afad526",
            "abae0496d8b848efb1c7f265944f34a2",
            "32e506a7998a4f79b31dd805306d1b5c"
          ]
        },
        "outputId": "33772978-f5d0-4b55-ba9b-467776104ccb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "734e83870e8845d696fcbe8132a375f5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "976b8a287dc14898a17236f13afad526"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2 points\n",
        "grader.grade(test_case_id = 'test_toys_games_final', answer = answer_toys_games_sdf)"
      ],
      "metadata": {
        "id": "iZx3RUv-Bb9u",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "46dac4c9-5d45-4588-ada5-1cb5cd7cf5ae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 2/2 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "This part is given to you. Combine the newly created `food_prod_sdf`, `toys_games_prod_sdf`, and `home_kitchen_prod_sdf` into a single dataframe called `products_sdf`. Combine `food_reviews_sdf`, `toys_games_reviews_sdf`, and `home_kitchen_reviews_sdf` into a single dataframe called `reviews_sdf`.\n",
        "\n",
        "***You do not need to edit the two cells below***."
      ],
      "metadata": {
        "id": "LZ9cw29N-FBV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "# Create SQL-accesible table\n",
        "food_prod_final_sdf.createOrReplaceTempView(\"food_prod_final\")\n",
        "home_kitchen_prod_final_sdf.createOrReplaceTempView(\"home_kitchen_prod_final\")\n",
        "toys_games_prod_final_sdf.createOrReplaceTempView(\"toys_games_prod_final\")\n",
        "\n",
        "# Declare SQL query to be excecuted\n",
        "query =\"\"\"(SELECT * FROM food_prod_final)\n",
        "           \n",
        "           UNION All\n",
        "           \n",
        "           (SELECT * FROM home_kitchen_prod_final)\n",
        "           \n",
        "           UNION All\n",
        "           \n",
        "           (SELECT * FROM toys_games_prod_final)\"\"\"\n",
        "\n",
        "# Save the output sdf of spark.sql() as products_sdf\n",
        "products_sdf = spark.sql(query)"
      ],
      "metadata": {
        "id": "rgLYCf8v7T8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "65a8fe86356146bda357f13d5348b3ef",
            "1f0876230fd3477daf4c35af81bb5b65",
            "c632002eb3ba44babc27a16ea4b9d13c"
          ]
        },
        "outputId": "3f79a009-069a-4593-9cbd-4b19eb8a17dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "65a8fe86356146bda357f13d5348b3ef"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "# Create SQL-accesible table\n",
        "food_reviews_sdf.createOrReplaceTempView(\"food_reviews_final\")\n",
        "home_kitchen_reviews_sdf.createOrReplaceTempView(\"home_kitchen_reviews_final\")\n",
        "toys_games_reviews_sdf.createOrReplaceTempView(\"toys_games_reviews_final\")\n",
        "\n",
        "# Declare SQL query to be excecuted\n",
        "query =\"\"\"(SELECT * FROM food_reviews_final)\n",
        "           \n",
        "           UNION All\n",
        "           \n",
        "           (SELECT * FROM home_kitchen_reviews_final)\n",
        "           \n",
        "           UNION All\n",
        "           \n",
        "           (SELECT * FROM toys_games_reviews_final)\"\"\"\n",
        "\n",
        "# Save the output sdf of spark.sql() as reviews_sdf\n",
        "reviews_sdf = spark.sql(query)\n",
        "\n"
      ],
      "metadata": {
        "id": "Etk5px6f7BAy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "0b2deaa1b6714b5f8df127076c8fc080",
            "536d1266c0404a0b82ecb7b27afaeb2a",
            "32cc6f6c35fb44638d46ae15147f8186"
          ]
        },
        "outputId": "7da24f55-6cd9-4870-9ea5-5c344cee7e6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b2deaa1b6714b5f8df127076c8fc080"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Join `reviews_sdf` and `products_sdf` on `asin` and name the resulting dataframe as `product_reviews_sdf`. Keep only the following columns from reviews_sdf: `reviewerID`, `asin`, `helpful`, `reviewText`, `overall`, `summary`, `reviewTime`, `unixReviewTime`. Keep the following columns from products_sdf: `title`, `category`, `price`."
      ],
      "metadata": {
        "id": "HNbDHz_Swuz1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create product_reviews_sdf\n",
        "reviews_sdf.createOrReplaceTempView(\"reviews\")\n",
        "products_sdf.createOrReplaceTempView(\"products\")\n",
        "query=\"\"\"\n",
        "SELECT R.reviewerID, R.asin, R.helpful, R.reviewText, R.overall, R.summary, R.reviewTime, R.unixReviewTime, P.title, P.category, P.price\n",
        "FROM reviews AS R\n",
        "INNER JOIN products AS P \n",
        "ON R.asin=P.asin\n",
        "\"\"\"\n",
        "\n",
        "product_reviews_sdf=spark.sql(query)"
      ],
      "metadata": {
        "id": "atRSOMSq-S-0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "7bafb8c72e9d44eebf5a2aff6b33ae56",
            "85e783308f604aa88675583bf05e18da",
            "f6b80c7e17304cc5a56797950ddd7835"
          ]
        },
        "outputId": "d791000d-4458-4aff-9286-371292ded596"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7bafb8c72e9d44eebf5a2aff6b33ae56"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "# Do not edit - run this after creating product_reviews_sdf\n",
        "product_reviews_sdf.createOrReplaceTempView(\"product_reviews\")\n",
        "ans_product_reviews_sdf=spark.sql(\"SELECT * FROM product_reviews ORDER BY asin ASC LIMIT 10\")"
      ],
      "metadata": {
        "id": "d0zNorGvBx6-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "8770d9b7a75541cc989297353fed2475",
            "9df659ee2edf400682c631dbc7027ee4",
            "366f2bb2d61d498183f1d4c1139007bc"
          ]
        },
        "outputId": "7bef5622-c7ed-4ccc-d3e8-f169d1de6974"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8770d9b7a75541cc989297353fed2475"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o ans_product_reviews_sdf"
      ],
      "metadata": {
        "id": "OnTE_pYGXb6z",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "b2d9014bdc2646e2a43acce3e4f7dd82",
            "89176493cffa4e8dbb892b1ef35d8868",
            "956d03b516ae40a49837182cf1fcdba0",
            "836b4fab1c9a41e498e7c7b7079159e0",
            "295f58937e17427fb74d976de20998a7",
            "041d3e2705b54d4391c88a9acef9341d"
          ]
        },
        "outputId": "c9674df2-55fa-484b-8d18-37bd6bd50bde"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b2d9014bdc2646e2a43acce3e4f7dd82"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "836b4fab1c9a41e498e7c7b7079159e0"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "# 3 points\n",
        "grader.grade(test_case_id = 'test_product_review', answer = ans_product_reviews_sdf)"
      ],
      "metadata": {
        "id": "nqflA81xVKoo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6d06bd7d-c458-4af2-b300-3e0147941913"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 3/3 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Step 2: Analysis! [26 pts]"
      ],
      "metadata": {
        "id": "CsLea-1Tyu0v"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "svOO4iLPist4"
      },
      "source": [
        "### Step 2.1: Overall Opinion [5 pts]\n",
        "\n",
        "We would like to find the average rating for products based on category. Using `product_reviews_sdf`, create `avg_rating_sdf` with schema `(category,avg_rating)`. Order your resulting dataframe by `avg_rating` (descending). Note that the column `overall` holds ratings for each product.\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create avg_rating_sdf\n",
        "product_reviews_sdf.createOrReplaceTempView(\"prod_reviews\")\n",
        "query =\"\"\"\n",
        "SELECT category,AVG(overall) AS avg_rating\n",
        "FROM prod_reviews\n",
        "GROUP BY category\n",
        "ORDER BY avg_rating DESC\n",
        "\"\"\"\n",
        "\n",
        "avg_rating_sdf=spark.sql(query)"
      ],
      "metadata": {
        "id": "JMXkywgqS8Iz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "b8c35410b9fb4b12b47e12fa3c04facc",
            "cc616afaf0354b3f8b3cccccb1c2904b",
            "37acc7d113d64fb3b284a380593bd669"
          ]
        },
        "outputId": "540bed43-08cc-46c7-95ba-6bc7fc525453"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b8c35410b9fb4b12b47e12fa3c04facc"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o avg_rating_sdf"
      ],
      "metadata": {
        "id": "2MN3JYNOThjW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "13fd488a60794829a54da03f4bf20eb1",
            "7fd40aa49a4a40ea9bdc1e4fa607a9b1",
            "79f3ed2b74c2411ba71aad6394618002",
            "764ee3d6b0a7412e89db41165840066f",
            "4912477113c646a1909220eeb5cde010",
            "f57987a0052b4be29fb22a1d51e1504f"
          ]
        },
        "outputId": "de89566c-4052-408e-efdc-26bc39bffab4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "13fd488a60794829a54da03f4bf20eb1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "764ee3d6b0a7412e89db41165840066f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "# 5 points\n",
        "grader.grade(test_case_id = 'test_avg_rating', answer = avg_rating_sdf)"
      ],
      "metadata": {
        "id": "6S6l60w-TjyC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f612cdda-2e63-451c-cca0-063207f8e888"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 5/5 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 2.2: The Category Changes [10 pts]\n",
        "\n",
        "We would like to find the percentage change from 2010 to 2011 in both average overall rating and average price for products based on category.\n",
        "\n",
        "*Hint: Use SUBSTR on the reviewTime field to retrieve the correct year.*\n",
        "\n",
        "Your task includes:\n",
        "\n",
        "\n",
        "*   Finding the average overall rating for each category for the years 2010, 2011\n",
        "*   Calculate the percentage change in rating for 2010 to 2011\n",
        "*   Finding the average price for each category for the years 2010, 2011\n",
        "*   Calculate the percentage change in price for 2010 to 2011\n",
        "*   Return an sdf with the category, change_rating, change_price sorted in descending alphabetical order of category name.\n",
        "\n",
        "Hint: Using multiple queries with intermediate dataframes may be helpful\n",
        "```\n",
        "+-------------------+--------------------+-------------+\n",
        "|           category|       change_rating|.change_price|\n",
        "+-------------------+--------------------|.------------|            \n",
        "+-------------------+--------------------+.------------|\n",
        "```"
      ],
      "metadata": {
        "id": "BW5HpURnC6X7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "#TODO: Create pct_change_sdf\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "dT1v_uDsrZNm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o pct_change_sdf"
      ],
      "metadata": {
        "id": "HWJCQBy7rcdN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 10 points\n",
        "grader.grade(test_case_id = 'test_pct_change', answer = pct_change_sdf)"
      ],
      "metadata": {
        "id": "60JVR1mirfRV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 2.3: The Popular Products [11 pts]\n",
        "\n",
        "When you navigate to the Amazon webpage, you are often recommended some bestsellers. These bestsellers are recommended to you based on several factors, one of them being positive ratings and reviews. Most popular products have tons of reviews and ratings from customers. We're going to look into popular products bought in a few categories to dive deeper into the consumer market culture. "
      ],
      "metadata": {
        "id": "MtjFKMD-C6va"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Step 2.3.1: The Terrific Toys [6 pts]\n",
        "\n",
        "From the product_reviews_sdf, find the 5 most commonly reviewed products around Christmas in the Toys and Games category. In this case, a product reviewed in the month of December (12), will count as 'around Christmas'.\n",
        "\n",
        "The resultant dataframe should contain the 5 most popular products and the number of reviews the product sorted in order of most to least popular in this format:\n",
        "\n",
        "\n",
        "```\n",
        "+-------------------+--------------------+\n",
        "|               asin|         num_reviews|\n",
        "+-------------------+--------------------|\n",
        "+-------------------+--------------------+\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "dYksV9qTC7ML"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create dec_toys_sdf\n",
        "popular_products_query =query =\"\"\"\n",
        "SELECT asin,COUNT(asin) AS num_reviews FROM prod_reviews WHERE category=\"Toys & Games\" AND (SUBSTR(reviewTime,1,3)=12)\n",
        "GROUP BY asin\n",
        "ORDER BY num_reviews DESC\n",
        "LIMIT 5\n",
        "\"\"\"\n",
        "\n",
        "dec_toys_sdf=spark.sql(query)"
      ],
      "metadata": {
        "id": "v6-01lW3tGrt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "b0fa1b4ff9c547fba6a3b7fe758a6230",
            "97a93b09ebb54368a3c0531624b1412d",
            "df67d3f890854fdcbc994691b9c05ad2"
          ]
        },
        "outputId": "ca551017-12ac-47cf-9934-988aaba6e285"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b0fa1b4ff9c547fba6a3b7fe758a6230"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o dec_toys_sdf"
      ],
      "metadata": {
        "id": "M142jjdntIMu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "719d1dafdb2e405aa3b61583db8bac41",
            "ed56f99bf66a4db8a113531b4c2e6c96",
            "5523fc2c85234866a430afced7d41261",
            "3950f444830440a786757848cfc5c288",
            "653aa1db75a24854bf74ad11532f7fa7",
            "b16ac7fe3b3941c8a0b8846068ab1632"
          ]
        },
        "outputId": "b877cc73-8246-48c5-a81c-8d010a0b1804"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "719d1dafdb2e405aa3b61583db8bac41"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3950f444830440a786757848cfc5c288"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "# 6 points\n",
        "grader.grade(test_case_id = 'test_dec_toys', answer = dec_toys_sdf)"
      ],
      "metadata": {
        "id": "HI-9TlCNtMYR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0724739-b9eb-4186-9489-58570ded77dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 6/6 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Step 2.3.2: Healthy Home [5 pts]"
      ],
      "metadata": {
        "id": "88HTvDoHDKQg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We are interested in finding the products that are most frequently reviewed within the Home and Kitchen category. Find the top three months in which products from the Home and Kitchen category are most frequently reviewed during the years 2010 to 2014 (inclusive). As before, the `SUBSTR` function in SQL will be helpful for you in your query.\n",
        "\n",
        "Return a dataframe with the schema `(month, num_reviews)` where `month` is the months in which products are most frequently reviewed (month values are integers: ex. 11 = November) and `num_reviews` is the number of reviews for that `month` during the years specified."
      ],
      "metadata": {
        "id": "dR0p39j9Ttqu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create home_kitch_three_sdf\n",
        "query =\"\"\"\n",
        "SELECT SUBSTR(reviewTime,1,2) as month,COUNT(reviewText) AS num_reviews\n",
        "FROM prod_reviews\n",
        "WHERE category=\"Home & Kitchen\" AND (SUBSTR(reviewTime, 7) BETWEEN  2010 AND 2014)\n",
        "GROUP BY SUBSTR(reviewTime,1,2)\n",
        "ORDER BY num_reviews DESC\n",
        "\"\"\"\n",
        "\n",
        "home_kitch_three_sdf = spark.sql(query)\n",
        "home_kitch_three_sdf.show()"
      ],
      "metadata": {
        "id": "cakzp4DUthj6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303,
          "referenced_widgets": [
            "24710dbd5dc6429a8754a034c62d0687",
            "88cc5b9fd4f740e0b57d488457a7dae5",
            "cdfd0276bb3a49b9aaddc555483d1f47"
          ]
        },
        "outputId": "b7c856e2-4c1f-42f2-dbea-02a6575fc7a8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "24710dbd5dc6429a8754a034c62d0687"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+-----------+\n",
            "|month|num_reviews|\n",
            "+-----+-----------+\n",
            "|   01|     414257|\n",
            "|   12|     329143|\n",
            "|   02|     318557|\n",
            "|   03|     318190|\n",
            "|   04|     280644|\n",
            "|   06|     277594|\n",
            "|   05|     277343|\n",
            "|   07|     266792|\n",
            "|   11|     206167|\n",
            "|   10|     188459|\n",
            "|   08|     180265|\n",
            "|   09|     172051|\n",
            "+-----+-----------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o home_kitch_three_sdf"
      ],
      "metadata": {
        "id": "MXis6SJ4yKHK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "5cb8987a8cb84302aa7a8da3ffcc021a",
            "f19a6074b2d3460b89ac482761a086f8",
            "001fc24932214ee9960d98a3788a2730",
            "3b4a83d309b645cf96c8fc2fab09be36",
            "b44bd77dda324e8b90277725743c6a41",
            "1c0b612fc0a548c3a8b23711852153b7"
          ]
        },
        "outputId": "9e320673-c0f7-49d1-f6b7-cb6c7aab1e13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5cb8987a8cb84302aa7a8da3ffcc021a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3b4a83d309b645cf96c8fc2fab09be36"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 5 points\n",
        "grader.grade(test_case_id = 'test_home_kitch_three', answer = home_kitch_three_sdf)"
      ],
      "metadata": {
        "id": "GTYWMVpsyexw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fd664ba8-0893-48fd-baed-2a5c6d46c539"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 5/5 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Step 3: The Riveting Reviews [30 pts]\n"
      ],
      "metadata": {
        "id": "9G1cFfEmDLit"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 3.1: The Short Summary [5 pts]\n",
        "\n",
        "You don't always have time to read an entire review. When you're about to buy a product you love, you may scroll through some reviews, eyeballing the summaries and how helpful the reviews are. Eyeballing a few reviews that comment on the usability of the product like \"cheap but works\" or \"effective\" or \"waste of money\" can help you make your decision. Your task is to count the number of reviews per category where the summary length is less than 10 characters and it is considered useful.\n",
        "\n",
        "```\n",
        "+-------------------+--------------------+\n",
        "|           category|     helpful_reviews|\n",
        "+-------------------+--------------------|\n",
        "+-------------------+--------------------+\n",
        "```\n",
        "\n",
        "\n",
        "Name this calculated count of reviews as helpful_reviews and rank categories in descending order of helpful_reviews.\n",
        "\n",
        "\n",
        "Useful rating is defined as a list - [no. of helpful votes, total no,of votes. For example, a [2,3] means 2 out of 3 others found this review helpful. So, to be considered useful a review must have 1 or more total votes, and 1 or more helpful votes."
      ],
      "metadata": {
        "id": "qxasT40_DORo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "\n",
        "#TODO: Create summary_sdf\n",
        "query=\"\"\"\n",
        "SELECT category,COUNT(helpful) as helpful_reviews\n",
        "FROM prod_reviews \n",
        "WHERE helpful[0]>0 AND helpful[1]>0 AND LENGTH(summary)<10\n",
        "GROUP BY category\n",
        "ORDER BY helpful_reviews DESC\n",
        "\"\"\"\n",
        "\n",
        "summary_sdf=spark.sql(query)\n",
        "summary_sdf.show()"
      ],
      "metadata": {
        "id": "PeKtZsamzw6g",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142,
          "referenced_widgets": [
            "ef15c05e2e2a4f73b60656712e44fa7b",
            "492cfaee7ae74661bad025b03ebaec85",
            "9ee4b47f40c84691b02aa247a7330b28"
          ]
        },
        "outputId": "1f4588f5-3c3d-4542-d173-730920c1c49a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ef15c05e2e2a4f73b60656712e44fa7b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+---------------+\n",
            "|      category|helpful_reviews|\n",
            "+--------------+---------------+\n",
            "|Home & Kitchen|         150574|\n",
            "|  Toys & Games|          75173|\n",
            "|          Food|          53525|\n",
            "+--------------+---------------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o summary_sdf"
      ],
      "metadata": {
        "id": "DcwtLJ9azwoy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "4691a7f37bba41ddb8e6a6d8061efb4f",
            "4e20ba4bb50543b9a4f11da4225c2282",
            "cfc424af93334191a4ae169a8e8b2b36",
            "6cb4268f54b14ffea28d30b83f0b8d55",
            "133e9e12e1c84f3abcbf623d8219bb6b",
            "6b1ee1fcba2c4331a9bd30a39cf8b6f9"
          ]
        },
        "outputId": "4b7a8be8-1783-4e12-f7c1-7a1d5c8d34ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4691a7f37bba41ddb8e6a6d8061efb4f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6cb4268f54b14ffea28d30b83f0b8d55"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 5 points\n",
        "grader.grade(test_case_id = 'test_summary', answer = summary_sdf)"
      ],
      "metadata": {
        "id": "DW-tKZr_zz7E",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "59014aa6-5400-477a-f67e-7322e365a3ce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 5/5 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 3.2: The Review Ratio [10 pts]\n",
        "\n",
        "Find how helpful a review is in terms of a ratio, and return the top 10 most helpful reviews, the corresponding reviewerID and the helpfulness ratio. The helpfulness ratio can be calulated as:\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        " $\\frac{Number\\ of \\ helpful \\ votes}{Total \\ number \\ of \\ votes}$\n",
        "\n",
        "\n",
        "The following columns are required in your top_10_helpful_sdf: reviewerID, reviewText, helpful_ratio. Make sure to sort the dataframe in descending order of helpful_ratio as well as descending order of reviewerID.\n",
        "\n",
        "**Note:** Ensure to only use useful ratings when computing the ratio"
      ],
      "metadata": {
        "id": "lnmqRSskDRvE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "#TODO: Create top10_helpful_sdf\n",
        "query =\"\"\"\n",
        "SELECT reviewerID,reviewText,(helpful[0]/helpful[1]) AS helpful_ratio\n",
        "FROM prod_reviews\n",
        "WHERE helpful[1]>0\n",
        "ORDER BY helpful_ratio DESC,reviewerID DESC\n",
        "LIMIT 10\n",
        "\"\"\"\n",
        "\n",
        "top10_helpful_sdf=spark.sql(query)\n",
        "top10_helpful_sdf.show()"
      ],
      "metadata": {
        "id": "sstIprXj1Z83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "4d360fe1aa8c47a9b78ccd50ad4dad91",
            "e9cd9f0d5314488c91c794176555361e",
            "9a6a6bd8b93349dfab569ff2d640d049"
          ]
        },
        "outputId": "03e74738-ccd4-477d-f774-e297299e10b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4d360fe1aa8c47a9b78ccd50ad4dad91"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+--------------------+-------------+\n",
            "|    reviewerID|          reviewText|helpful_ratio|\n",
            "+--------------+--------------------+-------------+\n",
            "|A30MXWKFWQBGQG|THIS PUPPET IS GR...|          2.0|\n",
            "|A2V0I904FH7ABY|It was almost a '...|          1.5|\n",
            "| AZZZZS5Z6MEWC|I got this and it...|          1.0|\n",
            "| AZZZOVIBXHGDR|I ordered this ce...|          1.0|\n",
            "| AZZZO4QCZROW8|I received this b...|          1.0|\n",
            "| AZZZKHVV482YT|I really like it ...|          1.0|\n",
            "| AZZZ871Y5ZC4F|These are deep, n...|          1.0|\n",
            "| AZZZ6G9WZTNNX|So awesome!!!  I ...|          1.0|\n",
            "| AZZZ3LGTCGUZF|Great gift.  Boug...|          1.0|\n",
            "| AZZZ3H7UKQZB1|I bought this set...|          1.0|\n",
            "+--------------+--------------------+-------------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o top10_helpful_sdf"
      ],
      "metadata": {
        "id": "3LlHp2NS1a6_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "5d40532a2626405ba08a5485578d012f",
            "f77ee9adde974f8791c38c5e8a5dd625",
            "bdb4a1b2346c40138656aab3ba5de232",
            "c97a0104645049ce97098f682e0d1025",
            "94f5e3516e3a4dc480fc6bb16107140f",
            "9c5192d9c7d14c1eaca3e1e3e5357064"
          ]
        },
        "outputId": "1f5bcace-e02c-4a2c-fc8c-1c951fdfc31c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5d40532a2626405ba08a5485578d012f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c97a0104645049ce97098f682e0d1025"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 10 points\n",
        "grader.grade(test_case_id = 'test_helpful_ratio', answer = top10_helpful_sdf)"
      ],
      "metadata": {
        "id": "QMPtc6A41fWU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d9a5e79-36ed-4d7c-ae03-45905b72e10c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 10/10 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 3.3: The Sensible Sentiment\n",
        "\n",
        "We would like to determine the sentiment of some of the reviews across the categories. Sentiment Analysis is used with textual data very regularly. A common application includes determing whether the review associated with a product is positive, negative or neutral. We are going to attempt to determine the sentiment associated with the top 10 most helpful reviews from the food category, based on the ratio calculated in the previous section. "
      ],
      "metadata": {
        "id": "rQR-vYBwDYjY"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Step 3.3.1: The Top Ten [5 pts]\n",
        "\n",
        "Retrieve the asin and reviewText for the top 10 most helpful reviews for the food category, where how helpful a review is determined by the ratio calculated in the previous question. Sort in descending order of helpfulness as well as reviewerID. "
      ],
      "metadata": {
        "id": "omdhaC5FDb88"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "#TODO : Create top10_sdf\n",
        "helpful_food_ratio_query =\"\"\"\n",
        "SELECT asin, reviewText \n",
        "FROM \n",
        "    (SELECT asin,reviewText,(helpful[0] / helpful[1]) AS helpful_ratio\n",
        "    FROM prod_reviews\n",
        "    WHERE helpful[1]>0 AND category=\"Food\"\n",
        "    ORDER BY helpful_ratio DESC,reviewerID DESC\n",
        "    LIMIT 10)\n",
        "\"\"\"\n",
        "\n",
        "top10_sdf=spark.sql(helpful_food_ratio_query)\n",
        "top10_sdf.show()"
      ],
      "metadata": {
        "id": "lfry_qaG1ixe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 267,
          "referenced_widgets": [
            "2370ecdf88ec4cf3b4e7e67465144936",
            "126a3bae93ba4b3ba4572d87c35f1ee3",
            "8770a0b180374a35be3f0372f7db3127"
          ]
        },
        "outputId": "6bf9a4e3-dd67-45f5-a2bb-fceb812ccde4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2370ecdf88ec4cf3b4e7e67465144936"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------------------+\n",
            "|      asin|          reviewText|\n",
            "+----------+--------------------+\n",
            "|B001EQ55RW|It was almost a '...|\n",
            "|B001EO5SB2|I ordered this ce...|\n",
            "|B00AUM3T7E|My dad really enj...|\n",
            "|B001EQ5EJQ|I use this for ev...|\n",
            "|B001EO6B7C|Baked in the oven...|\n",
            "|B005K4Q1YA|UNBELIEVABLE - CA...|\n",
            "|B004NY500O|This is not the f...|\n",
            "|B001E0HT2W|I love this syrup...|\n",
            "|B000FKMNUU|This product uses...|\n",
            "|B0015KAYN2|First of all, the...|\n",
            "+----------+--------------------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o top10_sdf"
      ],
      "metadata": {
        "id": "E50DWLFx1jxC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "effb549629cf4eb4aa46cbd838abe009",
            "5d1950bdaa7a4e73a42344ba9efbedfc",
            "b158dd2e6a574eb6a78d224c5b1ace69",
            "8bbcab2efd8d4f5ba2c2b07c8caf76b8",
            "8dffb39a819744e188af492d25cae2dd",
            "f805b139654d4bdeadd95473ec9505d8"
          ]
        },
        "outputId": "2f8afae2-c6c9-4564-ab46-7b4250476fca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "effb549629cf4eb4aa46cbd838abe009"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8bbcab2efd8d4f5ba2c2b07c8caf76b8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 5 points\n",
        "grader.grade(test_case_id = 'test_top10_helpful', answer = top10_sdf)"
      ],
      "metadata": {
        "id": "ZgDLT4IP1nSf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "989a05f8-b8bc-4b87-8bb3-969ed0a059bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 5/5 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Step 3.3.2: Positively Positive [10 pts]\n",
        "\n",
        "Recall from your previous assignment, you were asked to return a tokenized list of words (removing stop words and checking for alphabets). \n",
        " \n",
        "Using the pandas dataframe created from the previous section, create a new column called 'tokenized' which contains the tokens from the reviewText. (Hint: Use the apply function to call the custom tokenize function from HW2). \n",
        "\n",
        "Sentiment analysis is a a natural language processing (NLP) technique that attempts to identify the emotional tone behind a body of text. In this case, we will try to determine how posiitve a review is.\n",
        "\n",
        "\n",
        "We will be calculating the sentiment associated with each of these reviews. To read more about sentiment analysis, [click here](https://towardsdatascience.com/sentiment-analysis-concept-analysis-and-applications-6c94d6f58c17)."
      ],
      "metadata": {
        "id": "mSYdQxVlDgoZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('vader_lexicon')\n",
        "stopwords = set(stopwords.words('english'))"
      ],
      "metadata": {
        "id": "r54o6GQ_3dq6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d678dbb7-0f0e-40b9-8f7b-bc4cb859f4c6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Convert to dataframe & tokenize list \n",
        "#TODO: Add your custom tokenize content function \n",
        "def tokenize_content(content):\n",
        "  tokens=nltk.word_tokenize(content)\n",
        "  tokens=[t.lower() for t in tokens if t.isalpha()]\n",
        "  tokens=[t for t in tokens if t not in stopwords]\n",
        "\n",
        "  return tokens\n",
        "\n",
        "\n",
        "  \n",
        "#TODO: Apply your function to reviewText\n",
        "top10_tokenize_sdf=top10_sdf\n",
        "top10_tokenize_sdf[\"tokenized\"]=top10_tokenize_sdf[\"reviewText\"].apply(tokenize_content)"
      ],
      "metadata": {
        "id": "jO_nTqwY3mgD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "Run the cell below to understand how to the inbuilt Sentiment Analyzer works. As you can see below, the compound variable contains the overall sentiment of the sentence \"Wow, NLTK is really powerful!\".\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "a4mfalDpCKNr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#EXAMPLE CELL\n",
        "from nltk.sentiment import SentimentIntensityAnalyzer\n",
        "sia = SentimentIntensityAnalyzer()\n",
        "sia.polarity_scores(\"Wow, NLTK is really powerful!\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DJMLaQ8f3oFk",
        "outputId": "4f5bacfd-6350-4743-ca6d-e3d663c5329d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'neg': 0.0, 'neu': 0.295, 'pos': 0.705, 'compound': 0.8012}"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Your task is to now add a column to your dataframe called sentiment, and calculate the sentiment associated with each review. Sort this dataframe from highest sentiment to lowest.\n",
        "\n",
        "Now, moving back into SQL, we need you to run a query on the product_reviews_sdf to retrieve the titles of 2 products with the highest sentiment. Use the asins from the first two rows of your sorted pandas dataframes, and retrieve the just product titles. Save this result into top2_sentiment_sdf. Please do not hardcode the values of the titles, we expect an SQL approach to obtain the titles. However, you may hardcode ASIN values that you obtain from the top10_tokenize_sdf in your SQL query.\n",
        "\n",
        "You will be passing two results to the autograder : top10_tokenize_sdf (asin, reviewText, tokenized, sentiment columns), top2_sentiment_sdf (only title column)"
      ],
      "metadata": {
        "id": "J0S-6ZB0FCZB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sia = SentimentIntensityAnalyzer()\n",
        "def retrieve_sentiment(content):\n",
        "   return sia.polarity_scores(content)[\"compound\"]"
      ],
      "metadata": {
        "id": "NYDHG-1I3qBc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO: Create sentiment column and sort top10_tokenize_sdf by sentiment \n",
        "top10_sdf[\"sentiment\"] = top10_sdf[\"reviewText\"].apply(retrieve_sentiment)\n",
        "top10_tokenize_sdf = top10_sdf.sort_values(by='sentiment',ascending=False)\n",
        "top10_tokenize_sdf\n"
      ],
      "metadata": {
        "id": "F2QN2RbE5q8U",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "outputId": "6b4a9cdc-c5c9-45c6-9e04-4b40c7bbce18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         asin                                         reviewText  \\\n",
              "9  B0015KAYN2  First of all, the fragrance is subtle and heav...   \n",
              "2  B00AUM3T7E  My dad really enjoyed them! Mexico's food is b...   \n",
              "4  B001EO6B7C  Baked in the oven at 350F for 20 - 25 minutes....   \n",
              "0  B001EQ55RW  It was almost a 'love at first bite' - the per...   \n",
              "3  B001EQ5EJQ  I use this for everything--cooking, salads, fr...   \n",
              "7  B001E0HT2W  I love this syrup. It is my personal substitut...   \n",
              "8  B000FKMNUU  This product uses Sorbitol and Asparthame to a...   \n",
              "5  B005K4Q1YA  UNBELIEVABLE - CAN'T GO WITHOUT IT.....YUMMY !...   \n",
              "6  B004NY500O  This is not the flavor I was looking for. Like...   \n",
              "1  B001EO5SB2  I ordered this cereal based on the good review...   \n",
              "\n",
              "                                           tokenized  sentiment  \n",
              "9  [first, fragrance, subtle, heavenly, love, not...     0.9926  \n",
              "2  [dad, really, enjoyed, mexico, food, bland, gr...     0.9323  \n",
              "4  [baked, oven, minutes, taste, great, definitel...     0.9186  \n",
              "0  [almost, first, bite, perfectly, roasted, almo...     0.8941  \n",
              "3  [use, everything, cooking, salads, frying, add...     0.8316  \n",
              "7  [love, syrup, personal, substitute, bailey, ir...     0.7050  \n",
              "8  [product, uses, sorbitol, asparthame, add, swe...     0.6908  \n",
              "5  [unbelievable, ca, go, without, yummy, told, f...     0.6360  \n",
              "6  [flavor, looking, like, typical, bazooka, flav...     0.3612  \n",
              "1  [ordered, cereal, based, good, reviews, else, ...    -0.6451  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ee925010-2675-406d-9880-5557a22d3f09\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>asin</th>\n",
              "      <th>reviewText</th>\n",
              "      <th>tokenized</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>B0015KAYN2</td>\n",
              "      <td>First of all, the fragrance is subtle and heav...</td>\n",
              "      <td>[first, fragrance, subtle, heavenly, love, not...</td>\n",
              "      <td>0.9926</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>B00AUM3T7E</td>\n",
              "      <td>My dad really enjoyed them! Mexico's food is b...</td>\n",
              "      <td>[dad, really, enjoyed, mexico, food, bland, gr...</td>\n",
              "      <td>0.9323</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>B001EO6B7C</td>\n",
              "      <td>Baked in the oven at 350F for 20 - 25 minutes....</td>\n",
              "      <td>[baked, oven, minutes, taste, great, definitel...</td>\n",
              "      <td>0.9186</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>B001EQ55RW</td>\n",
              "      <td>It was almost a 'love at first bite' - the per...</td>\n",
              "      <td>[almost, first, bite, perfectly, roasted, almo...</td>\n",
              "      <td>0.8941</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>B001EQ5EJQ</td>\n",
              "      <td>I use this for everything--cooking, salads, fr...</td>\n",
              "      <td>[use, everything, cooking, salads, frying, add...</td>\n",
              "      <td>0.8316</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>B001E0HT2W</td>\n",
              "      <td>I love this syrup. It is my personal substitut...</td>\n",
              "      <td>[love, syrup, personal, substitute, bailey, ir...</td>\n",
              "      <td>0.7050</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>B000FKMNUU</td>\n",
              "      <td>This product uses Sorbitol and Asparthame to a...</td>\n",
              "      <td>[product, uses, sorbitol, asparthame, add, swe...</td>\n",
              "      <td>0.6908</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>B005K4Q1YA</td>\n",
              "      <td>UNBELIEVABLE - CAN'T GO WITHOUT IT.....YUMMY !...</td>\n",
              "      <td>[unbelievable, ca, go, without, yummy, told, f...</td>\n",
              "      <td>0.6360</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>B004NY500O</td>\n",
              "      <td>This is not the flavor I was looking for. Like...</td>\n",
              "      <td>[flavor, looking, like, typical, bazooka, flav...</td>\n",
              "      <td>0.3612</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>B001EO5SB2</td>\n",
              "      <td>I ordered this cereal based on the good review...</td>\n",
              "      <td>[ordered, cereal, based, good, reviews, else, ...</td>\n",
              "      <td>-0.6451</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ee925010-2675-406d-9880-5557a22d3f09')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ee925010-2675-406d-9880-5557a22d3f09 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ee925010-2675-406d-9880-5557a22d3f09');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%spark\n",
        "#TODO: Create top2_sentiment_sdf\n",
        "query=\"\"\"\n",
        "SELECT DISTINCT title\n",
        "FROM prod_reviews\n",
        "WHERE asin=\"B0015KAYN2\" OR asin=\"B00AUM3T7E\"\n",
        "\"\"\"\n",
        "\n",
        "top2_sentiment_sdf=spark.sql(query)\n",
        "top2_sentiment_sdf.show()"
      ],
      "metadata": {
        "id": "U_zAPYJw7-5x",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124,
          "referenced_widgets": [
            "2c824839e7724b6da705fac746adf6dc",
            "69afee623f99482aa3fc888adceaccb9",
            "03f53a716ab84aef95c759e3ee6987bb"
          ]
        },
        "outputId": "ea1789d2-0e58-4953-ff2d-7f40cbf2972c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2c824839e7724b6da705fac746adf6dc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+\n",
            "|               title|\n",
            "+--------------------+\n",
            "|Nature's Way Coco...|\n",
            "|Fritos Corn Chips...|\n",
            "+--------------------+"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%spark -o top2_sentiment_sdf"
      ],
      "metadata": {
        "id": "vGgMcQ2U8fNX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "aa5ca98974bb46f4bce724e44d837396",
            "85758bc889d24c4a9c105cc22e4ca1e8",
            "8d9a3445370e460aa462dc44f14cd22b",
            "5b9901221c6e4e889cfa91e8859549e9",
            "c7db6b275d6043638e6811177904d59c",
            "215450bc5f1f475198001f3d8432561f"
          ]
        },
        "outputId": "6c4aea10-4bfd-49cc-fcfe-e39e12089612"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "aa5ca98974bb46f4bce724e44d837396"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5b9901221c6e4e889cfa91e8859549e9"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# 10 points\n",
        "grader.grade(test_case_id = 'test_sentiment_positive', answer = (top2_sentiment_sdf,top10_tokenize_sdf))\n",
        "\n"
      ],
      "metadata": {
        "id": "ZPhfsFRc3sHG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad24bbe1-5587-4874-c2ef-e7bd7d47d08f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 10/10 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Step 4: Candy Land [26 pts]\n",
        "\n",
        "In this section, we will be working with a dataset from a poll where each row holds a person's favorite and second favorite candy/sweet. \n",
        "\n",
        "Let's introduce a fun little concept called the Bacon Number! The Bacon number of an actor or actress is the number of degrees of separation they have from actor Kevin Bacon, as defined by the game known as Six Degrees of Kevin Bacon. For example, Kevin Bacon's Bacon number is 0. If an actor works in a movie with Kevin Bacon, the actor's Bacon number is 1. If an actor works with an actor who worked with Kevin Bacon in a movie, the first actor's Bacon number is 2, and so forth.\n",
        "\n",
        "How do we implement the \"Super Bacon\" for our dataset though? We define this number as follows: if candy A is someone's favorite candy and candy B is that person's second favorite candy while candy B is someone else's favorite candy and candy C is their second favorite candy, then the super bacon of C with respect to A will be 2.\n",
        "\n",
        "Now to calculate this number, we'll use the concepts of graphs and BFS!"
      ],
      "metadata": {
        "id": "rGyXyvhBDr0-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pyspark"
      ],
      "metadata": {
        "id": "E_i8gCbAEuHW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "11d8d7c4-3c83-4682-9b7f-61cdeef65a6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.3.1.tar.gz (281.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 281.4 MB 34 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9.5\n",
            "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 25.7 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.3.1-py2.py3-none-any.whl size=281845513 sha256=7ca850718200df4e48babf04d9691f5d72fdd64b5b3c27df41dc094227cedc2b\n",
            "  Stored in directory: /root/.cache/pip/wheels/42/59/f5/79a5bf931714dcd201b26025347785f087370a10a3329a899c\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9.5 pyspark-3.3.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession\n",
        "\n",
        "appName = \"PySpark\"\n",
        "\n",
        "# Create Spark session\n",
        "spark = SparkSession.builder.appName(appName).getOrCreate()"
      ],
      "metadata": {
        "id": "s7AlDICWEt7e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 4.1 “Traversing” a Graph\n",
        "\n",
        "Let's review how BFS works!"
      ],
      "metadata": {
        "id": "zkJ2eyNrDtUl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from IPython.display import Image as I\n",
        "\n",
        "bfsgif =\\\n",
        "'https://upload.wikimedia.org/wikipedia/commons/5/5d/Breadth-First-S'+\\\n",
        "'earch-Algorithm.gif'\n",
        "dfsgif=\\\n",
        "'https://upload.wikimedia.org/wikipedia/commons/7/7f/Depth-First-Search.gif'"
      ],
      "metadata": {
        "id": "sH6pKfJJEzpu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "#### Step 4.1.1 Intro to Distributed Breadth-First Search\n",
        "\n",
        "\n",
        "To start off, we will be implementing a graph traversal algorithm known as Breadth First Search. It works in a way that's equivalent to how a stain spreads on a white t-shirt. Take a look at the graph below:\n",
        "\n",
        "<p align = \"center\">\n",
        "<img src = \"https://imgur.com/WU3AUwg.png\" width= \"600\" align =\"center\"/>\n",
        "\n",
        "* Consider starting BFS from point A (green). This is considered the starting frontier/singular origin node.\n",
        "* The first round of BFS would involve finding all the nodes directly reachable from A, namely B-F (blue circles). These blue nodes make up the next frontier at depth 1 away from our starting node A.\n",
        "* The second round would then be identifying the red nodes which are the neighbors of the blue nodes. Now, the red nodes all belong to a frontier 2 depth away from A. Note that node A is also a neighbor of a blue node. However, since it has already been visited, it does not get added to this frontier.\n",
        "\n",
        "This process continues until all the nodes in the graph have been visited.\n",
        "If you would like to learn more about BFS, we highly suggest looking [here](https://www.tutorialspoint.com/data_structures_algorithms/breadth_first_traversal.html).\n",
        "\n",
        "\n",
        "We will now be implementing **spark_bfs(G, N, d)**, our spark flavor of BFS that takes a graph **G**, a set of origin nodes **N**, and a max depth **d**.\n",
        "\n",
        "In order to write a successful BFS function, you are going to need to figure out \n",
        "1. how to keep track of nodes that we have visited\n",
        "2. how to properly find all the nodes at the next depth\n",
        "3. how to avoid cycles and ensure that we do not constantly loop through the same edges (take a look at J-K in the graph)"
      ],
      "metadata": {
        "id": "Oo3y1k4EDyod"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### [OPTIONAL/ADDITIONAL] BFS vs. DFS Animation \n",
        "Run the code cells below to understand the difference between depth and breadth first search! (Source: Wikimedia Commons)"
      ],
      "metadata": {
        "id": "NYjIiz_0E7Aw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Do not edit\n",
        "print('BFS:')\n",
        "I(url=bfsgif)"
      ],
      "metadata": {
        "id": "1wteWWS7E3KK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 539
        },
        "outputId": "7c777173-7f9e-41a7-b050-ab86a036198d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BFS:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<img src=\"https://upload.wikimedia.org/wikipedia/commons/5/5d/Breadth-First-Search-Algorithm.gif\"/>"
            ],
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {},
          "execution_count": 107
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Do not edit\n",
        "print('DFS:')\n",
        "I(url=dfsgif)"
      ],
      "metadata": {
        "id": "2-S7vJ9pE40i",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 539
        },
        "outputId": "65afed26-0027-49ef-ff93-87f6cf464f3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DFS:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<img src=\"https://upload.wikimedia.org/wikipedia/commons/7/7f/Depth-First-Search.gif\"/>"
            ],
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Step 4.1.2 Implement One Traversal [6 pts]\n",
        "\n",
        "To break down this process, let's think about how we would implement a single traversal of the graph. That is given the green node in the graph above, how are we going to get the blue nodes?\n",
        "\n",
        "\n",
        "Consider the simple graph below **which is different from the graph in the image above**:"
      ],
      "metadata": {
        "id": "pyuyXzNjEjEp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Do not edit\n",
        "spark.conf.set(\"spark.sql.execution.arrow.enabled\", \"true\")\n",
        "\n",
        "simple = [('A', 'B'),\n",
        "         ('A', 'C'),\n",
        "         ('A', 'D'),\n",
        "         ('C', 'F'),\n",
        "         ('F', 'A'),\n",
        "         ('B', 'G'),\n",
        "         ('G', 'H'),\n",
        "         ('D', 'E')]\n",
        "\n",
        "simple_dict = {'from_node': ['A', 'A', 'A', 'C', 'F', 'B', 'G', 'D'],\n",
        "       'to_node': ['B', 'C', 'D', 'F', 'A', 'G', 'H', 'E']}\n",
        "\n",
        "simple_graph_df = pd.DataFrame.from_dict(simple_dict)\n",
        "simple_graph_sdf = spark.createDataFrame(simple_graph_df)\n",
        "simple_graph_sdf.show()"
      ],
      "metadata": {
        "id": "74yrRK9AE-G5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1da0e952-328d-4805-8d60-0ce1da3ca0c8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+-------+\n",
            "|from_node|to_node|\n",
            "+---------+-------+\n",
            "|        A|      B|\n",
            "|        A|      C|\n",
            "|        A|      D|\n",
            "|        C|      F|\n",
            "|        F|      A|\n",
            "|        B|      G|\n",
            "|        G|      H|\n",
            "|        D|      E|\n",
            "+---------+-------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see, each row of this dataframe represents an edge between two nodes. Here, we are looking at a directed graph, which means that A-->B does not represent the same edge as B-->A.\n",
        "Let's define our starting node as follows:"
      ],
      "metadata": {
        "id": "KTbktZQ9FGox"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "smallOrig = [{'node': 'A'}]"
      ],
      "metadata": {
        "id": "POWS1z8hE-ym"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Then, bfs with graph G, starting from smallOrig to depth 1, or spark_bfs(G, smallOrig, 1) would output as follows:"
      ],
      "metadata": {
        "id": "xwgYZNZ-FHZg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Do not edit\n",
        "simple_1_round_dict = {'node': ['B', 'D', 'C', 'A'],\n",
        "       'distance': [1, 1, 1, 0]}\n",
        "simple_1_round_bfs_df = pd.DataFrame.from_dict(simple_1_round_dict)\n",
        "simple_1_round_bfs_sdf = spark.createDataFrame(simple_1_round_bfs_df)\n",
        "simple_1_round_bfs_sdf.show()"
      ],
      "metadata": {
        "id": "-hBlW2rPFKig",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1cec5b84-4860-45bb-88fe-fba0df7fc855"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+--------+\n",
            "|node|distance|\n",
            "+----+--------+\n",
            "|   B|       1|\n",
            "|   D|       1|\n",
            "|   C|       1|\n",
            "|   A|       0|\n",
            "+----+--------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As you can see, this dataframe logs each node with its corresponding distance away from A. Moreover, we also know that these nodes are **visited**. \n",
        "\n",
        "Hopefully, you can see how we can use our original graph and this new information to find the nodes at depth two. \n",
        "\n",
        "This is exactly what we will try to accomplish with **spark_bfs_1_round(visited_nodes)** which will ultimately be the inner function of **spark_bfs** that we use to perform exactly one traversal of a graph.\n",
        "\n",
        "**TODO**: Write **spark_bfs_1_round(visted_nodes)** that takes the currently dataframe of visited_nodes, performs one round of BFS, and returns an updated visited nodes dataframe. You should assume that a temporary sdf `G` already exists."
      ],
      "metadata": {
        "id": "15Qk24fGFO_U"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def spark_bfs_1_round(visited_nodes):\n",
        "\n",
        "  \"\"\"\n",
        "  :param visited_nodes: dataframe with columns node and distance\n",
        "  :return: dataframe of updated visuted nodes, with columns node and distance\n",
        "  \"\"\"\n",
        "  \n",
        "\n",
        "  \n",
        "\n",
        "  \n",
        "\n"
      ],
      "metadata": {
        "id": "OQYlGzw6FSms"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, run the inner function on simple_1_round_bfs_sdf (i.e. result of 1 round of BFS on the simple graph) and store the results in simple_bfs_result. This is ultimately what the output of BFS to depth 2 should look like."
      ],
      "metadata": {
        "id": "za3-poGjFaRy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO: Run spark_bfs_1_round on simple_1_round_bfs_sdf"
      ],
      "metadata": {
        "id": "IUTyEdbpFUVG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convert this result to Pandas, sort your dataframe by ***node*** ascending, and submit it to the autograder."
      ],
      "metadata": {
        "id": "evUUcJ0yFdhH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO: Convert simple_bfs_result to Pandas sorted by node"
      ],
      "metadata": {
        "id": "8wmaNu9YZIlH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "53a29da9-b0e6-449c-a896-be103b008f8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  node  distance\n",
              "3    A         0\n",
              "0    B         1\n",
              "2    C         1\n",
              "1    D         1"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0280ce48-2f93-45f0-8270-3c8d2fd7f67e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>node</th>\n",
              "      <th>distance</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>A</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>B</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>D</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0280ce48-2f93-45f0-8270-3c8d2fd7f67e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0280ce48-2f93-45f0-8270-3c8d2fd7f67e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0280ce48-2f93-45f0-8270-3c8d2fd7f67e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 206
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 6 points\n",
        "grader.grade(test_case_id = 'checksimpleBFS', answer = simple_bfs_test)"
      ],
      "metadata": {
        "id": "Z0IyMp1dFYEh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "abcf0ae1-36ee-429d-88ca-def8c3d8b748"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error: Test case failed. Test case function could not complete due to an error in your answer.\n",
            "Error Hint: 4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Step 4.1.3 Full BFS Implementation\n",
        "\n",
        "Now, we will fully implement **spark_bfs**. This function should iteratively call your implemented version of **spark_bfs_1_round** and ultimately return the output of this function at **max_depth**.\n",
        "\n",
        "You are also responsible for initializing the starting dataframe, that is converting the list of origin nodes into a spark dataframe with the nodes logged at distance 0.\n",
        "\n",
        "Consider the following: \n",
        "\n",
        "```\n",
        "schema = StructType([\n",
        "            StructField(\"node\", StringType(), True)\n",
        "        ])\n",
        "\n",
        "    my_sdf = spark.createDataFrame(origins, schema)\n",
        "```\n",
        "\n",
        "The schema ultimately specifies the structure of the Spark DataFrame with a string `node` column. It then calls **spark.createDataFrame** to map this schema to the **origins** nodes. Also, you are responsible for ensuring that a view of your graph is available within this function. (Note: you will also need to add in a distance column)\n",
        "\n",
        "**TODO:** implement **spark_bfs(G,origins,max_depth)** and run on **candy_graph_sdf** initalized in 4.3. Note: you may want to run tests on the **simple_graph** example as the `candy_graph_sdf` will take quite some time to run.\n",
        "\n",
        "These imports might be useful: \n",
        "`from pyspark.sql.types import StructType, StructField, StringType, IntegerType`"
      ],
      "metadata": {
        "id": "Nk2soBT2EoRl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.types import StructType, StructField, StringType, IntegerType\n",
        "# TODO: iterative search over undirected graph\n",
        "# Worth 5 points directly, but will be needed later\n",
        "\n",
        "def spark_bfs(G, origins, max_depth):\n",
        "  \"\"\" runs distributed BFS to a specified max depth\n",
        "\n",
        "  :param G: graph dataframe from 4.3\n",
        "  :param origins: list of origin nodes stored as {\"node\": nodeValue}\n",
        "  :param max_depth: integer value of max depth to run BFS to\n",
        "  :return: dataframe with columns node, distance of all visited nodes\n",
        "  \"\"\"\n",
        "\n",
        "  G.createOrReplaceTempView(\"G\")\n",
        "  query=\"SELECT * FROM G\"\n",
        "  G=spark.sql(query)\n",
        "  origins=spark.createDataFrame(origins) \n",
        "  origins.createOrReplaceTempView(\"origins\") \n",
        "  query=\"SELECT origins.node as node, 0 as distance FROM origins\"\n",
        "  visited_nodes=spark.sql(query) \n",
        "  visited_nodes.createOrReplaceTempView(\"visited_nodes\") \n",
        "  for i in range(max_depth):\n",
        "      visited_nodes=spark_bfs_1_round(visited_nodes)\n",
        "      visited_nodes.createOrReplaceTempView(\"visited_nodes\") \n",
        "  return visited_nodes"
      ],
      "metadata": {
        "id": "tIbkTuSTFhR4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test that this function works on the simple example."
      ],
      "metadata": {
        "id": "2LRo1B8HFjs1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "simple_bfs_iterative_result = spark_bfs(simple_graph_sdf, smallOrig, 3)\n",
        "simple_bfs_iterative_result.show()"
      ],
      "metadata": {
        "id": "rL9XjwyPFhyy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f50f84e0-f139-4a1e-c14f-80b5e6466edb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----+--------+\n",
            "|node|distance|\n",
            "+----+--------+\n",
            "|   F|       2|\n",
            "|   E|       2|\n",
            "|   B|       1|\n",
            "|   D|       1|\n",
            "|   C|       1|\n",
            "|   A|       0|\n",
            "|   G|       2|\n",
            "|   H|       3|\n",
            "+----+--------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 4.2: Popular Candies [5 pts]"
      ],
      "metadata": {
        "id": "FBIXSJQmFqZc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download the `candy_graph_extra.csv` dataset from [here](https://drive.google.com/file/d/1ZjpdZ9wPIMAXvHsr8aocg4IgYtkcLywp/view?usp=sharing), and upload it to the content folder of this Colab notebook."
      ],
      "metadata": {
        "id": "QkpLsfunGKmc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "It's time to take a look at our candy dataset. Let's load the data to **candy_sdf** and see how it looks."
      ],
      "metadata": {
        "id": "J9tlk9skXegN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "candy_sdf = spark.read.csv(\"/content/candy_graph_extra.csv\", header=True)"
      ],
      "metadata": {
        "id": "F7EsOfD5NMea"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "candy_sdf.show()"
      ],
      "metadata": {
        "id": "bN-1SaYENlD-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d6ca69b4-262e-42d6-b70e-5b908bf91ca3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+--------------------+--------------------+\n",
            "|_c0|          FirstCandy|         SecondCandy|\n",
            "+---+--------------------+--------------------+\n",
            "|  0|  SweetTarts 200 pcs|York Peppermint P...|\n",
            "|  1|Starburst Sour Chews|  Haribo Gummy Bears|\n",
            "|  2|Weetabix Crispy M...|Starburst Sour Chews|\n",
            "|  3|Starburst Sour Chews|         Gummy Worms|\n",
            "|  4|        Milk Duds 24|Twizzlers Strawbe...|\n",
            "|  5|Starburst Sour Chews|Henry Goode's Str...|\n",
            "|  6|Henry Goode's Str...|        Milk Duds 24|\n",
            "|  7|    Hershey's Kisses|York Peppermint P...|\n",
            "|  8|Reeses Peanut But...|Starburst Sour Chews|\n",
            "|  9|   Skittles 200g Bag|Kit Kat Chunky Mu...|\n",
            "| 10|     M&Ms Party Pack|Butterfinger Peca...|\n",
            "| 11|Kit Kat Chunky Mu...|Snowball Cotton C...|\n",
            "| 12|Reeses Peanut But...|Starburst Sour Chews|\n",
            "| 13|Butterfinger Peca...|Gobstopper Chewy ...|\n",
            "| 14|Kit Kat Chunky Mu...|Henry Goode's Str...|\n",
            "| 15|Starburst Sour Chews|Weetabix Crispy M...|\n",
            "| 16|Starburst Sour Chews|Twizzlers Strawbe...|\n",
            "| 17|        3 Musketeers|Jelly Belly jelly...|\n",
            "| 18|Starburst Sour Chews|Easter Bunny cand...|\n",
            "| 19|    Hershey's Kisses|Reeses Peanut But...|\n",
            "+---+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's take a look at which candies were the most popular. \n",
        "\n",
        "***TODO***: Count the number of times that each candy was chosen as their favorite (each candy appears in the FirstCandy column). Return a dataframe with schema (FirstCandy,Votes) where Votes is the number of times that each candy was chosen as a person's favorite. Order your dataframe by Votes."
      ],
      "metadata": {
        "id": "JPftgXHoXneJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO\n",
        "\n",
        "candy_sdf.createOrReplaceTempView(\"candy_votes\")\n",
        "\n",
        "\n",
        "count_top_df=spark.sql(\"\"\"\n",
        "SELECT FirstCandy,COUNT(FirstCandy) AS Votes\n",
        "FROM candy_votes\n",
        "GROUP BY FirstCandy\n",
        "ORDER BY Votes DESC\"\"\")"
      ],
      "metadata": {
        "id": "idqHrl-BNrH5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convert the dataframe to pandas and pass it to the grader."
      ],
      "metadata": {
        "id": "Qpjvw2j3I9Nt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "count_df=count_top_df.toPandas()"
      ],
      "metadata": {
        "id": "-UV5KUtO_RlD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grader.grade(test_case_id = 'checkCandyCount', answer = count_df)"
      ],
      "metadata": {
        "id": "tF6l4PLF_OKB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "180efaf2-65f9-4321-e3ba-52d74f390872"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 3/3 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now that we have the Candy dataset, let's convert it to a graph sdf just like the one we had in section 4.1 (P.S. it's not as hard as it sounds).\n",
        "\n",
        "**TODO:** Create **candy_graph_sdf** that has the columns *from_node* and *to_node*. from_node has all the entries from the *FirstCandy* column and to_node has all the entries from the *SecondCandy* column."
      ],
      "metadata": {
        "id": "Y7-sQIg9Ydze"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# TODO\n",
        "\n",
        "candy_graph_sdf=spark.sql(\"\"\"\n",
        "SELECT FirstCandy\n",
        "AS from_node, SecondCandy AS to_node FROM candy\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "pY37jvGkOBeN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grader.grade(test_case_id = 'checkCandyGraph', answer = candy_graph_sdf.toPandas())"
      ],
      "metadata": {
        "id": "_K7U-RF4_eSv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ccf3d2b-dd69-45dd-9128-08f2dcaf3397"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 2/2 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 4.3: The Super Bacon Search [15 pts]"
      ],
      "metadata": {
        "id": "duQ9JsnL_hqh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the previous section, we found out that the Starburst Sour Chews and Kit Kat Chunky Mug Egg  have been the most popular candies for this group of customers. So let's find out the Super Bacons of candies with respect to Starburst Sour Chews."
      ],
      "metadata": {
        "id": "oQyp_EjeM_16"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, we'll create an origin node using Starburst Sour Chews."
      ],
      "metadata": {
        "id": "W4iQD5nJOrJy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "orig = [{'node': 'Starburst Sour Chews'}]"
      ],
      "metadata": {
        "id": "Bk8yuvpsOErc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, run the **spark_bfs()** function on **teams_graph_sdf** using this origin node and a max depth of 5. Store the result in **bfs_5**."
      ],
      "metadata": {
        "id": "rw_lX0nPPoYT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "bfs_5 = spark_bfs(candy_graph_sdf, orig, 5)\n"
      ],
      "metadata": {
        "id": "cXukRdc-Ox0u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bfs_5.show()"
      ],
      "metadata": {
        "id": "BJX5QrQxnXeH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd1e2fef-ee83-4e33-db61-4d70ae07976b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+--------+\n",
            "|                node|distance|\n",
            "+--------------------+--------+\n",
            "|Twizzlers Strawbe...|       1|\n",
            "|        Milk Duds 24|       2|\n",
            "|Kit Kat Chunky Mu...|       4|\n",
            "|   Skittles 200g Bag|       3|\n",
            "|Nerds Rope Assort...|       5|\n",
            "|Jelly Belly jelly...|       4|\n",
            "|  SweetTarts 200 pcs|       4|\n",
            "|Snowball Cotton C...|       5|\n",
            "|Henry Goode's Str...|       1|\n",
            "|Stephen's Gourmet...|       2|\n",
            "|         Gummy Worms|       1|\n",
            "|Weetabix Crispy M...|       1|\n",
            "|Starburst Sour Chews|       0|\n",
            "|Strawberry Cherry...|       5|\n",
            "|Easter Bunny cand...|       1|\n",
            "|York Peppermint P...|       5|\n",
            "|  Haribo Gummy Bears|       1|\n",
            "|     Sour Patch Kids|       5|\n",
            "+--------------------+--------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convert the result to pandas, store it to answer_df and pass this to the grader."
      ],
      "metadata": {
        "id": "gkmBQnFJQJjz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "answer_df = bfs_5.toPandas()"
      ],
      "metadata": {
        "id": "_bFCML1YnYmo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grader.grade(test_case_id = 'checkBFS', answer = answer_df)"
      ],
      "metadata": {
        "id": "-1IVMn3Q_rob",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4c94b63e-4fc2-402c-f59b-5077857f1409"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Correct! You earned 15/15 points. You are a star!\n",
            "\n",
            "Your submission has been successfully recorded in the gradebook.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Voila! We're all done."
      ],
      "metadata": {
        "id": "3dePHd9CQW-C"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EHVFHPVIrvu1"
      },
      "source": [
        "# HW Submission\n",
        "\n",
        "Before you submit on Gradescope (you must submit your notebook to receive credit):\n",
        "\n",
        "\n",
        "1.   Restart and Run-All to make sure there's nothing wrong with your notebook\n",
        "2.   **Double check that you have the correct PennID (all numbers) in the autograder**. \n",
        "3. Make sure you've run all the PennGrader cells\n",
        "4. Go to the \"File\" tab at the top left, and click \"Download .ipynb\" and then \"Download .py\".  **Rename** the files to \"homework3.ipynb\" and \"homework3.py\" respectively and upload them to Gradescope \n",
        "\n",
        "**Let the course staff know ASAP if you have any issues submitting, but otherwise best of luck!**"
      ]
    }
  ]
}