{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "A100",
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8a807c2780cf4a65ad92dfa1ead0e439": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_021bf8d789b74e3d899a99eeb82a0036",
              "IPY_MODEL_f7aeed7c0692419591ca57bffee51739",
              "IPY_MODEL_d7f2da7e3beb42dfaf31dc69eb7a597e"
            ],
            "layout": "IPY_MODEL_c7b913c695fc448c8fc7b31f7098f636"
          }
        },
        "021bf8d789b74e3d899a99eeb82a0036": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a59325353a1d4902b03f5a37c0293934",
            "placeholder": "​",
            "style": "IPY_MODEL_6cd3d0fc462c4b14bbe6be92b8e6103e",
            "value": "100%"
          }
        },
        "f7aeed7c0692419591ca57bffee51739": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0cfdc6a2fa49499e9f0ce350b88dfbc4",
            "max": 3288938,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a9c70dae97424ca290a75e10d24baf52",
            "value": 3288938
          }
        },
        "d7f2da7e3beb42dfaf31dc69eb7a597e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0fc3fee01b04ffc922096ee18b6cb0f",
            "placeholder": "​",
            "style": "IPY_MODEL_9142d6e9793648a9909e5ce3dabf415b",
            "value": " 3.29M/3.29M [00:00&lt;00:00, 9.24MiB/s]"
          }
        },
        "c7b913c695fc448c8fc7b31f7098f636": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a59325353a1d4902b03f5a37c0293934": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6cd3d0fc462c4b14bbe6be92b8e6103e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0cfdc6a2fa49499e9f0ce350b88dfbc4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9c70dae97424ca290a75e10d24baf52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f0fc3fee01b04ffc922096ee18b6cb0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9142d6e9793648a9909e5ce3dabf415b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0ee9d77e31c343d2b226c9e1c370899a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cefbf3ec854e463eac3047d7f2366a9b",
              "IPY_MODEL_5d5a04539517439c9a47f725bbfca641",
              "IPY_MODEL_487df5716a1f42b9ad08a05300882448"
            ],
            "layout": "IPY_MODEL_f6f95cf2eb1f43cdbb1be5a1ffc19621"
          }
        },
        "cefbf3ec854e463eac3047d7f2366a9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8b2545d7d94490b8df9d8a63d7fde2d",
            "placeholder": "​",
            "style": "IPY_MODEL_18b7438e17ef4851aa9c8d0dfc52f2d5",
            "value": "Optimization Progress: 100%"
          }
        },
        "5d5a04539517439c9a47f725bbfca641": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_939ebe26aaf74df78c81c255aefcc7e5",
            "max": 20,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c5677148554a4e65b74ae9fc95ae4c37",
            "value": 20
          }
        },
        "487df5716a1f42b9ad08a05300882448": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_328be8dce5cf4b2a8418e7693f1829c0",
            "placeholder": "​",
            "style": "IPY_MODEL_48d9d503aebd44578cb545fdb4b28776",
            "value": " 40/40 [02:13&lt;00:00,  2.60s/pipeline]"
          }
        },
        "f6f95cf2eb1f43cdbb1be5a1ffc19621": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "a8b2545d7d94490b8df9d8a63d7fde2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18b7438e17ef4851aa9c8d0dfc52f2d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "939ebe26aaf74df78c81c255aefcc7e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5677148554a4e65b74ae9fc95ae4c37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "328be8dce5cf4b2a8418e7693f1829c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "48d9d503aebd44578cb545fdb4b28776": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "873451c433c1441796e0f2c582b54edc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_12bd969e54e64b17b22957ca3da533fa",
              "IPY_MODEL_4dd1615acf4645d7861226a3da60ae27",
              "IPY_MODEL_b5c015fcd9bd4461885e5e5c7f49fa9a"
            ],
            "layout": "IPY_MODEL_c46835dbd92c49d6aef630a6d74f8588"
          }
        },
        "12bd969e54e64b17b22957ca3da533fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8bf87a1e0740469cbea6f0c76dd37735",
            "placeholder": "​",
            "style": "IPY_MODEL_cc4bbcf9f0c44cd69117dc6251954049",
            "value": "Optimization Progress: 100%"
          }
        },
        "4dd1615acf4645d7861226a3da60ae27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b988186cafdf4da2a30bcff8a6a00f70",
            "max": 20,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f998e026eba0488288515e8b69eacc96",
            "value": 20
          }
        },
        "b5c015fcd9bd4461885e5e5c7f49fa9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0df60724539b4708a9323dbb479fa044",
            "placeholder": "​",
            "style": "IPY_MODEL_dad976092fff4859aaf73eb9d85aa01b",
            "value": " 120/120 [07:16&lt;00:00,  3.02s/pipeline]"
          }
        },
        "c46835dbd92c49d6aef630a6d74f8588": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "8bf87a1e0740469cbea6f0c76dd37735": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc4bbcf9f0c44cd69117dc6251954049": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b988186cafdf4da2a30bcff8a6a00f70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f998e026eba0488288515e8b69eacc96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0df60724539b4708a9323dbb479fa044": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dad976092fff4859aaf73eb9d85aa01b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4b8ebee65b2e4789bce617bedb691cb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a2a4e23c74fb4892be3f9edd122efd35",
              "IPY_MODEL_1abae52fd84841759c034f9fb78c65a3",
              "IPY_MODEL_36359ad021e64e6eb733ae740d987e68"
            ],
            "layout": "IPY_MODEL_80d36d8a650f42d8a990e04014e937a1"
          }
        },
        "a2a4e23c74fb4892be3f9edd122efd35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_751c1855804a48ff9f0d6496a5e29506",
            "placeholder": "​",
            "style": "IPY_MODEL_4749064df71f48f4b947ba208acbac26",
            "value": "Optimization Progress: 100%"
          }
        },
        "1abae52fd84841759c034f9fb78c65a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_14db8797959a4253a36dd334f4826a00",
            "max": 20,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_94e3a14430f64fa9ab318f0e6c37a883",
            "value": 20
          }
        },
        "36359ad021e64e6eb733ae740d987e68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b974c27ffe7444ab7f9683167a6988c",
            "placeholder": "​",
            "style": "IPY_MODEL_e13925b2a40c4dcf993c62f64d332272",
            "value": " 120/120 [08:06&lt;00:00,  4.12s/pipeline]"
          }
        },
        "80d36d8a650f42d8a990e04014e937a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "751c1855804a48ff9f0d6496a5e29506": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4749064df71f48f4b947ba208acbac26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14db8797959a4253a36dd334f4826a00": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94e3a14430f64fa9ab318f0e6c37a883": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9b974c27ffe7444ab7f9683167a6988c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e13925b2a40c4dcf993c62f64d332272": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d143776d4b464a648b17ae8e280ecc39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_16a6911e48b142cf83401adb3788792f",
              "IPY_MODEL_68f7ddcbed594c8c8a511c20be8d4acd",
              "IPY_MODEL_4c3ff39abc4b4f0baeee0538ce75d76f"
            ],
            "layout": "IPY_MODEL_1f8d747cea354729af6b532fb2a23a97"
          }
        },
        "16a6911e48b142cf83401adb3788792f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f634d0b3e4c34f058ab52077ccb97407",
            "placeholder": "​",
            "style": "IPY_MODEL_5df15aaaa49b425e827b57a427479f89",
            "value": "Optimization Progress: 100%"
          }
        },
        "68f7ddcbed594c8c8a511c20be8d4acd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_11ac4cb2fd7641e98779b027d47f2696",
            "max": 20,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d7a0b39588cb421492dde89492430a84",
            "value": 20
          }
        },
        "4c3ff39abc4b4f0baeee0538ce75d76f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe991ac34e224d55b9c27ca1bad2808b",
            "placeholder": "​",
            "style": "IPY_MODEL_ba7425e6aedf46f5adce1a7e8e2a7c4d",
            "value": " 120/120 [07:32&lt;00:00,  4.69s/pipeline]"
          }
        },
        "1f8d747cea354729af6b532fb2a23a97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "f634d0b3e4c34f058ab52077ccb97407": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5df15aaaa49b425e827b57a427479f89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "11ac4cb2fd7641e98779b027d47f2696": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7a0b39588cb421492dde89492430a84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fe991ac34e224d55b9c27ca1bad2808b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba7425e6aedf46f5adce1a7e8e2a7c4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2b1a6384b01945e8b5422f8033b35f3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ac9c8d71d04244faa9bc62060455bf18",
              "IPY_MODEL_1fc73e630cf24f478dfbb3b8f2761383",
              "IPY_MODEL_6e5336aa80094ee3826ba89335fbfd77"
            ],
            "layout": "IPY_MODEL_e8d44a1dded246ae94cd3413a216be4d"
          }
        },
        "ac9c8d71d04244faa9bc62060455bf18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d77493bbfb9d4197bc83de151f5398c7",
            "placeholder": "​",
            "style": "IPY_MODEL_627dcda5ea37455da054403897787bf2",
            "value": "100%"
          }
        },
        "1fc73e630cf24f478dfbb3b8f2761383": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a6d24f5b014f4202805c31486fc656cb",
            "max": 8039037,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8ab240c04e6e4ec5bc14918d47deffdf",
            "value": 8039037
          }
        },
        "6e5336aa80094ee3826ba89335fbfd77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_823df5cd8416469a9d826b118f741d6d",
            "placeholder": "​",
            "style": "IPY_MODEL_2ac41eef5c6248c6ae773df951bd82d2",
            "value": " 8.04M/8.04M [00:00&lt;00:00, 43.4MiB/s]"
          }
        },
        "e8d44a1dded246ae94cd3413a216be4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d77493bbfb9d4197bc83de151f5398c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "627dcda5ea37455da054403897787bf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a6d24f5b014f4202805c31486fc656cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ab240c04e6e4ec5bc14918d47deffdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "823df5cd8416469a9d826b118f741d6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ac41eef5c6248c6ae773df951bd82d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b188bad6827047b4aa8d57facea61b3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7b85fe2ab1314a27bf6f2715bcf2c253",
              "IPY_MODEL_7c6ae3b2016a4974ad5d80a36fa59c5f",
              "IPY_MODEL_72ad35f43fcb414f85a72baea10ceaee"
            ],
            "layout": "IPY_MODEL_f6087d8ae5584ed8805f20655307c694"
          }
        },
        "7b85fe2ab1314a27bf6f2715bcf2c253": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3f1706f0a6643abbe02debdccc80d16",
            "placeholder": "​",
            "style": "IPY_MODEL_b2023b49e067416a8712a878187ae26a",
            "value": "100%"
          }
        },
        "7c6ae3b2016a4974ad5d80a36fa59c5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d836a994fe684de792bffea9d47097b0",
            "max": 1716416,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e72b3600beb84517a48e51558e681ecb",
            "value": 1716416
          }
        },
        "72ad35f43fcb414f85a72baea10ceaee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ea0250740634945be80f5352bee2b28",
            "placeholder": "​",
            "style": "IPY_MODEL_c1290b54ba5d405fad098180f83b7a5a",
            "value": " 1.72M/1.72M [00:00&lt;00:00, 30.2MiB/s]"
          }
        },
        "f6087d8ae5584ed8805f20655307c694": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3f1706f0a6643abbe02debdccc80d16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2023b49e067416a8712a878187ae26a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d836a994fe684de792bffea9d47097b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e72b3600beb84517a48e51558e681ecb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4ea0250740634945be80f5352bee2b28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c1290b54ba5d405fad098180f83b7a5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a720f2592e3d4bef98a4c6f5077f277f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d2ae2ec449c14266b009f213df117da2",
              "IPY_MODEL_979b4554ad0b474b883040d9bbd51274",
              "IPY_MODEL_be36aefa1b3e4ea6a776e02c082c9958"
            ],
            "layout": "IPY_MODEL_7fc5e7ea894245579a1abdeed4492954"
          }
        },
        "d2ae2ec449c14266b009f213df117da2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_88b18cc3b9cf4eaa8edc5b16aaf28821",
            "placeholder": "​",
            "style": "IPY_MODEL_d5935c9201ed471ab2385df6e607380c",
            "value": "Optimization Progress: 100%"
          }
        },
        "979b4554ad0b474b883040d9bbd51274": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4aa0b28b2aa44951aad3bfe37fa20051",
            "max": 20,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_43490d899aed4338823374634d05c9f1",
            "value": 20
          }
        },
        "be36aefa1b3e4ea6a776e02c082c9958": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_512a81b06bd54811bd8b9f80958fa7a4",
            "placeholder": "​",
            "style": "IPY_MODEL_201d44e85f374a06929ae8db5851d61b",
            "value": " 120/120 [08:06&lt;00:00,  3.01s/pipeline]"
          }
        },
        "7fc5e7ea894245579a1abdeed4492954": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "88b18cc3b9cf4eaa8edc5b16aaf28821": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5935c9201ed471ab2385df6e607380c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4aa0b28b2aa44951aad3bfe37fa20051": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "43490d899aed4338823374634d05c9f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "512a81b06bd54811bd8b9f80958fa7a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "201d44e85f374a06929ae8db5851d61b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# AutoML with TPOT using Adversary Emulation (C2) as datagen - Supervised and Deep Learning with Genetic Programming"
      ],
      "metadata": {
        "id": "YJeseKzwjOVm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        ">[AutoML with TPOT using Adversary Emulation (C2) as datagen - Supervised and Deep Learning with Genetic Programming](#scrollTo=YJeseKzwjOVm)\n",
        "\n",
        ">[Installations](#scrollTo=Fv4KCLz9j4Zc)\n",
        "\n",
        ">[Tpot can use RAPIDS for better GPU support](#scrollTo=67jTk-_N42-A)\n",
        "\n",
        ">[Initialize GitHub authentication (token)](#scrollTo=OQTe2IE95P7c)\n",
        "\n",
        ">[Data download: captured Sysmon logs from the AE lab](#scrollTo=oYVNy4rNojZc)\n",
        "\n",
        ">>[ML development Objectives](#scrollTo=oYVNy4rNojZc)\n",
        "\n",
        ">[Download pre-trained BPE Tokenizer from the project](#scrollTo=Uyn6jyZvMsQI)\n",
        "\n",
        ">[ETL and labeling](#scrollTo=qM4Ym_5Zon4w)\n",
        "\n",
        ">[Sorting the data to get a timeline](#scrollTo=tKBQ5qsaBH02)\n",
        "\n",
        ">[Denoised vector generation through message filtering](#scrollTo=cYjpfMq1qKb1)\n",
        "\n",
        ">[Data exploration: create extra tables for labelling](#scrollTo=E_THJfYFmgLc)\n",
        "\n",
        ">[Qualitative behaviour analysis: temporary folder on Windows 10](#scrollTo=IuK7hfb_mq8e)\n",
        "\n",
        ">[Quantitative behaviour analysis: How many \"bad\" conditions did this yield?](#scrollTo=mVu5858Xm8qU)\n",
        "\n",
        ">[Store the data as CSV and JSON](#scrollTo=BGPg1RDmxMVE)\n",
        "\n",
        ">[Labeling based on Python](#scrollTo=GgFNrU80xQsF)\n",
        "\n",
        ">[Check of the Adversary Emulation data: a simulation dataset for InfoSec](#scrollTo=MES2a9BdnkU4)\n",
        "\n",
        ">[Vectorization with Linformer](#scrollTo=9-1dkT2mQHvJ)\n",
        "\n",
        ">[Example messages from the two classes (for a single-label classifier)](#scrollTo=91khWgQT3DVa)\n",
        "\n",
        ">[Example messages from the two classes (for a single-label classifier) (pandas framework code)](#scrollTo=ZY-8c7Q_3mEg)\n",
        "\n",
        ">[Vectorize text column in the DataFrame](#scrollTo=h4cuBLWrSNVs)\n",
        "\n",
        ">[The following code performs the high-dimensional vector projection with the Linformer model](#scrollTo=kSVB7YH-4czH)\n",
        "\n",
        ">[Saving the vectorized dataset as Parquet](#scrollTo=ScKF9UeB66LK)\n",
        "\n",
        ">[Import the data](#scrollTo=7LTDH-8RecLy)\n",
        "\n",
        ">[Kaggle download: log2ml dataset with Adversary Emulation data](#scrollTo=zmJzQP08sFfd)\n",
        "\n",
        ">[Import the parquet file into polars with pandas](#scrollTo=g7Tzv9jXsLWs)\n",
        "\n",
        ">[Load data into pandas](#scrollTo=_LydloD7sczq)\n",
        "\n",
        ">[AutoML with TPOT (Supervised Learning)](#scrollTo=ZqGIRBU6aE3U)\n",
        "\n",
        ">[Sparse vector data](#scrollTo=0Xxcb4ITMAH_)\n",
        "\n",
        ">[TPOT for Supervised Learning with Algo stats](#scrollTo=Gf4Fu4cDs4pr)\n",
        "\n",
        ">[TPOT Algo Stats: Supervised Learning](#scrollTo=KYJ1AkAStACV)\n",
        "\n",
        ">[Model export: TPOT's best pipeline, Supervised](#scrollTo=vM6TVwHYtKNa)\n",
        "\n",
        ">[Evaluation of the model](#scrollTo=lqXIlzZ0u7oN)\n",
        "\n",
        ">[Handling Imbalanced Data](#scrollTo=RG5bRw2lt0GD)\n",
        "\n",
        ">[Testing the predictions](#scrollTo=zCJ3jHyZQIS_)\n",
        "\n",
        ">[Save the Supervised Learning Model (joblib)](#scrollTo=YVPPGLSA6uZ-)\n",
        "\n",
        ">[Confusion matrix](#scrollTo=l1u70hEaAYgD)\n",
        "\n",
        ">[CV scores](#scrollTo=0jtLBuvnAeND)\n",
        "\n",
        ">[Grid search optimization](#scrollTo=_XUIxpRdKzf9)\n",
        "\n",
        ">[Tpot (Supervised)](#scrollTo=ZWRH59wKLaAX)\n",
        "\n",
        ">[Building a NN classifier with Tpot for an imbalanced dataset](#scrollTo=PEWJBJqf15m-)\n",
        "\n",
        ">[Using cuML for GPU accelerated model gen](#scrollTo=QJQXi06FdUDN)\n",
        "\n"
      ],
      "metadata": {
        "colab_type": "toc",
        "id": "rObtGEu1R21B"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "For long-running tasks it's possible to keep Colab from disconnecting (July 2024):\n",
        "\n",
        "https://gist.github.com/pouyaardehkhani/29a59270801a209d4960e2aefe648bbc"
      ],
      "metadata": {
        "id": "wUrhwXsDkETs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Installation of depdencies (Google Colab)\n",
        "\n",
        "The installation is mostly automated.\n",
        "\n",
        "A file in the same directory named \"thesis_ro\" will be required, which should contain your GitHub read-only token.\n",
        "\n",
        "The file has one line:\n",
        "\n",
        "\n",
        "`echo \"GITHUB_PERSONAL_ACCESS_TOKEN=\"ghp_...\" > thesis_ro`\n",
        "\n",
        "Colab has a set of tools installed by default. If this Notebook is used outside of Colab, additional depenencies may be required."
      ],
      "metadata": {
        "id": "Fv4KCLz9j4Zc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ayxWRSxzgwh_"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "import os\n",
        "import subprocess\n",
        "\n",
        "\"\"\"\n",
        "ATTENTION: this runs a shell script without further confirmations.\n",
        "Modifications to the systems may be required.\n",
        "\n",
        "Please use this in a lab environemt.\n",
        "\"\"\"\n",
        "\n",
        "IN_COLAB = 'google.colab' in sys.modules\n",
        "\n",
        "if not IN_COLAB:\n",
        "  pass\n",
        "\n",
        "else:\n",
        "  subprocess.run('''\n",
        "    source <(curl -s https://raw.githubusercontent.com/norandom/log2ml/main/dependencies/install.sh)\n",
        "    ''',\n",
        "    shell=True, check=True, executable='/bin/bash')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tpot can use RAPIDS for improvived GPU acceleration\n",
        "\n",
        "https://medium.com/rapids-ai/faster-automl-with-tpot-and-rapids-758455cd89e5\n",
        "\n",
        "https://docs.rapids.ai/api/cuml/stable/\n",
        "\n",
        "https://docs.rapids.ai/deployment/stable/platforms/colab/\n",
        "\n",
        "* cuml should be version 24. This is only relevant for some accelerated GPU-only models, which Tpot can generate with the help of DEAP (GP)\n",
        "\n",
        "https://deap.readthedocs.io/en/master/"
      ],
      "metadata": {
        "id": "67jTk-_N42-A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import cuml\n",
        "cuml.__version__"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "0Gu0pYEO4pxy",
        "outputId": "9bc36d8d-db34-4f45-c3ef-b5dd913db591"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'24.04.00'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Initialize GitHub authentication (token)\n",
        "\n",
        "GitHub's API doesn't allow anonymous release downloads. We can use a read-only token.\n",
        "\n",
        "https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28"
      ],
      "metadata": {
        "id": "OQTe2IE95P7c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from dotenv import load_dotenv\n",
        "import os\n",
        "\n",
        "\"\"\"\n",
        "Please ensure that you place the token into the file \"thesis_ro\".\n",
        "Use the same folder, or modify the code.\n",
        "Do not echo the token, and save the notebook.\n",
        "Otherwise the token leaks.\n",
        "\"\"\"\n",
        "\n",
        "\n",
        "load_dotenv(\"thesis_ro\", verbose=True)  # take environment variables from the file\n",
        "token = os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')\n",
        "if len(token) > 0:\n",
        "  print(\"ok\")\n",
        "else:\n",
        "  print(\"no token\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vyKWa35bkFcG",
        "outputId": "39c78aeb-bcd9-4bfe-aa8e-d56b4e819eec"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ok\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data download: captured Sysmon logs from the AE lab\n",
        "\n",
        "These samples contain Sysmon log activity of Dropper Malware (C2 Dropper, MS Excel VBA, Covenant).\n",
        "\n",
        "No full Adversary Emulation campaigns with post-exploitation, just the Dropper itself.\n",
        "\n",
        "The set contains security agent telemetry (aka sysmon lohs) of  1000 documents, some malicious and some not. These are imbalanced datasets.\n",
        "\n",
        "https://github.com/norandom/log2ml/releases/tag/lab\n",
        "\n",
        "## ML development Objectives\n",
        "\n",
        "1. **Explore**: how does the VBA Excel malware behave?  \n",
        "2. **Classify**: which ones are malicious?\n",
        "3. **Evaluate**: Can ML help to find out?\n",
        "4. **Engineer**: can Tpot autogen the ML models?"
      ],
      "metadata": {
        "id": "oYVNy4rNojZc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from github import Github\n",
        "import requests\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "\n",
        "def get_specific_file_from_tagged_release(token, repo_name, tag_name, filename):\n",
        "    g = Github(token)\n",
        "    repo = g.get_repo(repo_name)\n",
        "    releases = repo.get_releases()\n",
        "\n",
        "    for release in releases:\n",
        "        if release.tag_name == tag_name:\n",
        "            for asset in release.get_assets():\n",
        "                if asset.name == filename:\n",
        "                    return asset.url\n",
        "    print(\"File not found. Try get_specific_file_from_latest_release() instead.\")\n",
        "    return None\n",
        "\n",
        "def get_specific_file_from_latest_release(token, repo_name, filename):\n",
        "    g = Github(token)\n",
        "    repo = g.get_repo(repo_name)\n",
        "    release = repo.get_latest_release()\n",
        "\n",
        "    for asset in release.get_assets():\n",
        "        if asset.name == filename:\n",
        "            return asset.url  # Use asset.url which points to API URL needing headers\n",
        "\n",
        "def download_file(url, token, save_path):\n",
        "    headers = {'Authorization': f'token {token}', 'Accept': 'application/octet-stream'}\n",
        "    # First request to handle GitHub's redirection and authentication properly\n",
        "    with requests.get(url, headers=headers, stream=True) as initial_response:\n",
        "        initial_response.raise_for_status()  # Ensure the initial request is successful\n",
        "        # Follow redirection if necessary, maintaining headers\n",
        "        if initial_response.history:\n",
        "            url = initial_response.url  # Updated URL after redirection\n",
        "\n",
        "        # Now, proceed with downloading the file\n",
        "        with requests.get(url, headers=headers, stream=True) as response:\n",
        "            response.raise_for_status()\n",
        "            total_size_in_bytes = int(response.headers.get('content-length', 0))\n",
        "            block_size = 1024\n",
        "\n",
        "            progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)\n",
        "            with open(save_path, 'wb') as file:\n",
        "                for data in response.iter_content(block_size):\n",
        "                    progress_bar.update(len(data))\n",
        "                    file.write(data)\n",
        "            progress_bar.close()\n",
        "\n",
        "            if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:\n",
        "                print(\"ERROR, something went wrong\")\n",
        "            else:\n",
        "                print(f\"File downloaded successfully and saved as {save_path}\")"
      ],
      "metadata": {
        "id": "5EEoa3gUmFpn"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Your GitHub token\n",
        "github_token = token\n",
        "\n",
        "# Repository name\n",
        "repository_name = \"norandom/log2ml\""
      ],
      "metadata": {
        "id": "wwb-mj2pUAz_"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# File name to search for\n",
        "file_name = \"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024.csv\"\n",
        "\n",
        "# Get the download URL of the specific file\n",
        "# download_url = get_specific_file_from_latest_release(github_token, repository_name, file_name)\n",
        "download_url = get_specific_file_from_tagged_release(github_token, repository_name, \"lab\", file_name)\n",
        "print(download_url)\n",
        "\n",
        "if download_url:\n",
        "    local_file_path = file_name\n",
        "    download_file(download_url, github_token, local_file_path)\n",
        "else:\n",
        "    print(\"File not found.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85,
          "referenced_widgets": [
            "2b1a6384b01945e8b5422f8033b35f3e",
            "ac9c8d71d04244faa9bc62060455bf18",
            "1fc73e630cf24f478dfbb3b8f2761383",
            "6e5336aa80094ee3826ba89335fbfd77",
            "e8d44a1dded246ae94cd3413a216be4d",
            "d77493bbfb9d4197bc83de151f5398c7",
            "627dcda5ea37455da054403897787bf2",
            "a6d24f5b014f4202805c31486fc656cb",
            "8ab240c04e6e4ec5bc14918d47deffdf",
            "823df5cd8416469a9d826b118f741d6d",
            "2ac41eef5c6248c6ae773df951bd82d2"
          ]
        },
        "id": "yn8bBu9tT5_W",
        "outputId": "00ecb757-1f83-49a5-a8fd-02fe855961ea"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://api.github.com/repos/norandom/log2ml/releases/assets/182477524\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0.00/8.04M [00:00<?, ?iB/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2b1a6384b01945e8b5422f8033b35f3e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File downloaded successfully and saved as lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024.csv\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Download pre-trained BPE Tokenizer from the project\n",
        "\n",
        "The BPE tokenizer is used for the Sysmon messages, not for the entire JSON or CSV files.\n",
        "\n",
        "https://towardsdatascience.com/byte-pair-encoding-subword-based-tokenization-algorithm-77828a70bee0\n",
        "\n",
        "The tokenizer has been trained on the messages."
      ],
      "metadata": {
        "id": "Uyn6jyZvMsQI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# File name to search for\n",
        "file_name = \"log_tokenizer.json\"\n",
        "\n",
        "# Get the download URL of the specific file\n",
        "# download_url = get_specific_file_from_latest_release(github_token, repository_name, file_name)\n",
        "download_url = get_specific_file_from_tagged_release(github_token, repository_name, \"lab\", file_name)\n",
        "print(download_url)\n",
        "\n",
        "if download_url:\n",
        "    local_file_path = file_name\n",
        "    download_file(download_url, github_token, local_file_path)\n",
        "else:\n",
        "    print(\"File not found.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85,
          "referenced_widgets": [
            "b188bad6827047b4aa8d57facea61b3c",
            "7b85fe2ab1314a27bf6f2715bcf2c253",
            "7c6ae3b2016a4974ad5d80a36fa59c5f",
            "72ad35f43fcb414f85a72baea10ceaee",
            "f6087d8ae5584ed8805f20655307c694",
            "c3f1706f0a6643abbe02debdccc80d16",
            "b2023b49e067416a8712a878187ae26a",
            "d836a994fe684de792bffea9d47097b0",
            "e72b3600beb84517a48e51558e681ecb",
            "4ea0250740634945be80f5352bee2b28",
            "c1290b54ba5d405fad098180f83b7a5a"
          ]
        },
        "id": "PA-8JXLyn4Uv",
        "outputId": "67390f4f-b3a0-4f44-f80c-c9afc008910f"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://api.github.com/repos/norandom/log2ml/releases/assets/182534935\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0.00/1.72M [00:00<?, ?iB/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b188bad6827047b4aa8d57facea61b3c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File downloaded successfully and saved as log_tokenizer.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ETL and labeling\n",
        "\n",
        "This uses Polars as an ETL (**Extract Transform Load**) tool. This concerns in-memory processing of data tables, based on DataFrames. These are very popular in DataScience, because many tools can consume DataFrames.\n",
        "\n",
        "https://pola.rs/\n",
        "\n",
        "Pandas is used later, because it's more established and compatible to Tpot, which is used for Genetic Programming AutoML; for hyperparameter optimization and automated feature selection.\n",
        "\n",
        "https://pandas.pydata.org/"
      ],
      "metadata": {
        "id": "qM4Ym_5Zon4w"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import polars as pl\n",
        "\n",
        "csv_file_path = 'lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024.csv'\n",
        "\n",
        "# Load the CSV file into a DataFrame\n",
        "df = pl.read_csv(csv_file_path)\n",
        "\n",
        "# Show the DataFrame to confirm it's loaded correctly\n",
        "print(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SCKEWW-aoNDQ",
        "outputId": "ecb8cbda-d26e-44a0-f5ee-5cd1ea57a969"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (13_455, 7)\n",
            "┌──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐\n",
            "│ @timestamp   ┆ host.hostna ┆ host.ip     ┆ log.level   ┆ winlog.even ┆ winlog.task ┆ message     │\n",
            "│ ---          ┆ me          ┆ ---         ┆ ---         ┆ t_id        ┆ ---         ┆ ---         │\n",
            "│ str          ┆ ---         ┆ str         ┆ str         ┆ ---         ┆ str         ┆ str         │\n",
            "│              ┆ str         ┆             ┆             ┆ i64         ┆             ┆             │\n",
            "╞══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡\n",
            "│ 2024-07-28T1 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 13          ┆ Registry    ┆ Registry    │\n",
            "│ 5:09:33.716Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ value set   ┆ value set:  │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ (rule:      ┆ RuleName:   │\n",
            "│              ┆             ┆             ┆             ┆             ┆ Regist…     ┆ In…         │\n",
            "│ 2024-07-28T1 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 1           ┆ Process     ┆ Process     │\n",
            "│ 5:09:33.932Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ Create      ┆ Create:     │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ (rule:      ┆ RuleName: - │\n",
            "│              ┆             ┆             ┆             ┆             ┆ ProcessCre… ┆ UtcT…       │\n",
            "│ 2024-07-28T1 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 11          ┆ File        ┆ File        │\n",
            "│ 5:09:34.154Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ created     ┆ created:    │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ (rule:      ┆ RuleName:   │\n",
            "│              ┆             ┆             ┆             ┆             ┆ FileCreate) ┆ EXE         │\n",
            "│              ┆             ┆             ┆             ┆             ┆             ┆ UtcT…       │\n",
            "│ 2024-07-28T1 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 11          ┆ File        ┆ File        │\n",
            "│ 5:09:34.154Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ created     ┆ created:    │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ (rule:      ┆ RuleName:   │\n",
            "│              ┆             ┆             ┆             ┆             ┆ FileCreate) ┆ EXE         │\n",
            "│              ┆             ┆             ┆             ┆             ┆             ┆ UtcT…       │\n",
            "│ …            ┆ …           ┆ …           ┆ …           ┆ …           ┆ …           ┆ …           │\n",
            "│ 2024-07-28T2 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 5           ┆ Process     ┆ Process     │\n",
            "│ 1:14:59.942Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ terminated  ┆ terminated: │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ (rule:      ┆ RuleName: - │\n",
            "│              ┆             ┆             ┆             ┆             ┆ Proces…     ┆ …           │\n",
            "│ 2024-07-28T2 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 22          ┆ Dns query   ┆ Dns query:  │\n",
            "│ 1:14:46.543Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ (rule:      ┆ RuleName: - │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ DnsQuery)   ┆ UtcTime: …  │\n",
            "│ 2024-07-28T2 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 10          ┆ Process     ┆ Process     │\n",
            "│ 1:15:26.859Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ accessed    ┆ accessed:   │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ (rule:      ┆ RuleName: - │\n",
            "│              ┆             ┆             ┆             ┆             ┆ ProcessA…   ┆ Ut…         │\n",
            "│ 2024-07-28T2 ┆ win10       ┆ fe80::c1af: ┆ information ┆ 22          ┆ Dns query   ┆ Dns query:  │\n",
            "│ 1:15:29.929Z ┆             ┆ 35de:6006:d ┆             ┆             ┆ (rule:      ┆ RuleName: - │\n",
            "│              ┆             ┆ 4cf         ┆             ┆             ┆ DnsQuery)   ┆ UtcTime: …  │\n",
            "└──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sorting the data to get a timeline\n",
        "\n",
        "The data was received from the Elasticsearch DB.\n",
        "\n",
        "By defeault Elasticsearch sorts the data based on when it was received. Not based on the log timestamp."
      ],
      "metadata": {
        "id": "tKBQ5qsaBH02"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# sort data\n",
        "\n",
        "# Convert @timestamp to timestamp type\n",
        "df = df.with_columns(\n",
        "    pl.col(\"@timestamp\").str.to_datetime()\n",
        ")\n",
        "\n",
        "\n",
        "df = df.sort(\"@timestamp\")"
      ],
      "metadata": {
        "id": "Dzc4ZwG0NcMQ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Denoised vector generation through message filtering\n",
        "\n",
        "Here system specific strings get removed. This improves the transferability.\n",
        "Sysmon creates very detailed logs.\n",
        "\n",
        "The filtered_message column contains the \"denoised\" data."
      ],
      "metadata": {
        "id": "cYjpfMq1qKb1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_keyword_lines(batch, keywords):\n",
        "    def modify_message(message):\n",
        "        lines = message.split('\\n')\n",
        "        processed_lines = []\n",
        "        processed_keywords = set()\n",
        "\n",
        "        for line in lines:\n",
        "            processed = False\n",
        "            for keyword in keywords:\n",
        "                if line.startswith(f\"{keyword}:\") and keyword not in processed_keywords:\n",
        "                    processed_keywords.add(keyword)\n",
        "                    if keyword == 'User':\n",
        "                        parts = line.split('\\\\')\n",
        "                        if len(parts) > 1:\n",
        "                            processed_lines.append(f\"User: {parts[1]}\")\n",
        "                    elif keyword in ['SourceHostname', 'DestinationHostname']:\n",
        "                        parts = line.split(':', 1)\n",
        "                        if len(parts) > 1:\n",
        "                            hostname = parts[1].strip().split('.')[0]\n",
        "                            processed_lines.append(f\"{keyword}: {hostname}\")\n",
        "                    elif keyword == 'QueryName':\n",
        "                        parts = line.split(':', 1)\n",
        "                        if len(parts) > 1:\n",
        "                            domain = parts[1].strip().split('.')\n",
        "                            if len(domain) > 0:\n",
        "                                processed_lines.append(f\"QueryName: {domain[0]}\")\n",
        "                    else:\n",
        "                        processed_lines.append(line)\n",
        "                    processed = True\n",
        "                    break\n",
        "            if not processed:\n",
        "                processed_lines.append(line)\n",
        "\n",
        "        return '\\n'.join(processed_lines)\n",
        "\n",
        "    return batch.map_elements(modify_message, return_dtype=pl.Utf8)\n",
        "\n",
        "# Keywords to filter or process\n",
        "keywords_to_filter = [\"UtcTime\", \"SourceProcessGUID\", \"ProcessGuid\", \"TargetProcessGUID\", \"TargetObject\", \"FileVersion\", \"Hashes\", \"LogonGuid\", \"LogonId\", \"CreationUtcTime\", \"User\", \"ParentProcessGuid\", \"SourceHostname\", \"DestinationHostname\", \"QueryName\"]\n",
        "\n",
        "# Apply the transformation to the 'message' column using map_batches\n",
        "df_f = df.with_columns(\n",
        "    pl.col(\"message\").map_batches(lambda batch: remove_keyword_lines(batch, keywords_to_filter), return_dtype=pl.Utf8).alias(\"filtered_message\")\n",
        ")\n",
        "\n",
        "# Show the DataFrame to confirm it's loaded correctly\n",
        "print(df_f)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_CJUqGQUqFew",
        "outputId": "0fe25e96-12d1-419f-bb6a-626312d2441b"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (13_455, 8)\n",
            "┌────────────┬────────────┬────────────┬───────────┬───────────┬───────────┬───────────┬───────────┐\n",
            "│ @timestamp ┆ host.hostn ┆ host.ip    ┆ log.level ┆ winlog.ev ┆ winlog.ta ┆ message   ┆ filtered_ │\n",
            "│ ---        ┆ ame        ┆ ---        ┆ ---       ┆ ent_id    ┆ sk        ┆ ---       ┆ message   │\n",
            "│ datetime[μ ┆ ---        ┆ str        ┆ str       ┆ ---       ┆ ---       ┆ str       ┆ ---       │\n",
            "│ s, UTC]    ┆ str        ┆            ┆           ┆ i64       ┆ str       ┆           ┆ str       │\n",
            "╞════════════╪════════════╪════════════╪═══════════╪═══════════╪═══════════╪═══════════╪═══════════╡\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 3         ┆ Network   ┆ Network   ┆ Network   │\n",
            "│ 15:08:24.2 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ connectio ┆ connectio ┆ connectio │\n",
            "│ 77 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ n         ┆ n         ┆ n         │\n",
            "│            ┆            ┆            ┆           ┆           ┆ detected  ┆ detected: ┆ detected: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ (rul…     ┆ Rul…      ┆ Rul…      │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 3         ┆ Network   ┆ Network   ┆ Network   │\n",
            "│ 15:08:24.4 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ connectio ┆ connectio ┆ connectio │\n",
            "│ 88 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ n         ┆ n         ┆ n         │\n",
            "│            ┆            ┆            ┆           ┆           ┆ detected  ┆ detected: ┆ detected: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ (rul…     ┆ Rul…      ┆ Rul…      │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 10        ┆ Process   ┆ Process   ┆ Process   │\n",
            "│ 15:08:25.0 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ accessed  ┆ accessed: ┆ accessed: │\n",
            "│ 05 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ (rule:    ┆ RuleName: ┆ RuleName: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ ProcessA… ┆ -         ┆ -         │\n",
            "│            ┆            ┆            ┆           ┆           ┆           ┆ Ut…       ┆ Ut…       │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 10        ┆ Process   ┆ Process   ┆ Process   │\n",
            "│ 15:08:25.0 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ accessed  ┆ accessed: ┆ accessed: │\n",
            "│ 05 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ (rule:    ┆ RuleName: ┆ RuleName: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ ProcessA… ┆ -         ┆ -         │\n",
            "│            ┆            ┆            ┆           ┆           ┆           ┆ Ut…       ┆ Ut…       │\n",
            "│ …          ┆ …          ┆ …          ┆ …         ┆ …         ┆ …         ┆ …         ┆ …         │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 10        ┆ Process   ┆ Process   ┆ Process   │\n",
            "│ 23:35:53.0 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ accessed  ┆ accessed: ┆ accessed: │\n",
            "│ 54 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ (rule:    ┆ RuleName: ┆ RuleName: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ ProcessA… ┆ -         ┆ -         │\n",
            "│            ┆            ┆            ┆           ┆           ┆           ┆ Ut…       ┆ Ut…       │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 10        ┆ Process   ┆ Process   ┆ Process   │\n",
            "│ 23:35:54.1 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ accessed  ┆ accessed: ┆ accessed: │\n",
            "│ 33 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ (rule:    ┆ RuleName: ┆ RuleName: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ ProcessA… ┆ -         ┆ -         │\n",
            "│            ┆            ┆            ┆           ┆           ┆           ┆ Ut…       ┆ Ut…       │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 10        ┆ Process   ┆ Process   ┆ Process   │\n",
            "│ 23:35:54.1 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ accessed  ┆ accessed: ┆ accessed: │\n",
            "│ 33 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ (rule:    ┆ RuleName: ┆ RuleName: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ ProcessA… ┆ -         ┆ -         │\n",
            "│            ┆            ┆            ┆           ┆           ┆           ┆ Ut…       ┆ Ut…       │\n",
            "│ 2024-07-28 ┆ win10      ┆ fe80::c1af ┆ informati ┆ 1         ┆ Process   ┆ Process   ┆ Process   │\n",
            "│ 23:41:55.3 ┆            ┆ :35de:6006 ┆ on        ┆           ┆ Create    ┆ Create:   ┆ Create:   │\n",
            "│ 01 UTC     ┆            ┆ :d4cf      ┆           ┆           ┆ (rule:    ┆ RuleName: ┆ RuleName: │\n",
            "│            ┆            ┆            ┆           ┆           ┆ ProcessCr ┆ -         ┆ -         │\n",
            "│            ┆            ┆            ┆           ┆           ┆ e…        ┆ UtcT…     ┆ UtcT…     │\n",
            "└────────────┴────────────┴────────────┴───────────┴───────────┴───────────┴───────────┴───────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data exploration: create extra tables for labelling\n",
        "\n",
        "The following data exploration is part of the analytic process, which aims to produce labelled data for the Supervised Learning approach."
      ],
      "metadata": {
        "id": "E_THJfYFmgLc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "\n",
        "# Extract relevant information using regular expressions\n",
        "def extract_info(text):\n",
        "    image = re.search(r\"Image: (.*?\\.exe)\", text, re.IGNORECASE)\n",
        "    target_filename = re.search(r\"TargetFilename: (.*?\\.exe)\", text, re.IGNORECASE)\n",
        "    parent_image = re.search(r\"ParentImage: (.*?\\.exe)\", text, re.IGNORECASE)\n",
        "\n",
        "    return {\n",
        "        \"image\": image.group(1) if image else \"\",\n",
        "        \"target_filename\": target_filename.group(1) if target_filename else \"\",\n",
        "        \"parent_image\": parent_image.group(1).split(\"\\\\\")[-1] if parent_image else \"\",\n",
        "        \"text\": text\n",
        "    }\n",
        "\n",
        "# Apply extraction to the Polars DataFrame using map_elements\n",
        "df_f = df_f.with_columns(\n",
        "    pl.col(\"filtered_message\").map_elements(lambda x: extract_info(x), return_dtype=pl.Object).alias(\"extracted_info\")\n",
        ")\n",
        "\n",
        "# Extract fields from the extracted_info column using map_elements with return_dtype\n",
        "df_f = df_f.with_columns(\n",
        "    pl.col(\"extracted_info\").map_elements(lambda x: x['image'], return_dtype=pl.Utf8).alias(\"image\"),\n",
        "    pl.col(\"extracted_info\").map_elements(lambda x: x['target_filename'], return_dtype=pl.Utf8).alias(\"target_filename\"),\n",
        "    pl.col(\"extracted_info\").map_elements(lambda x: x['parent_image'], return_dtype=pl.Utf8).alias(\"parent_image\"),\n",
        "    pl.col(\"extracted_info\").map_elements(lambda x: x['text'], return_dtype=pl.Utf8).alias(\"text\")\n",
        ").drop(\"extracted_info\")\n",
        "\n",
        "print(df_f.head())\n",
        "\n",
        "# Print the unique values in the parent_image column\n",
        "print(df_f[\"parent_image\"].value_counts())\n",
        "print(df_f[\"target_filename\"].value_counts())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WBy3Rqj_orz_",
        "outputId": "60dbbef7-0753-45af-b6df-379880b63347"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (5, 12)\n",
            "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
            "│ @timestam ┆ host.host ┆ host.ip   ┆ log.level ┆ … ┆ image     ┆ target_fi ┆ parent_im ┆ text     │\n",
            "│ p         ┆ name      ┆ ---       ┆ ---       ┆   ┆ ---       ┆ lename    ┆ age       ┆ ---      │\n",
            "│ ---       ┆ ---       ┆ str       ┆ str       ┆   ┆ str       ┆ ---       ┆ ---       ┆ str      │\n",
            "│ datetime[ ┆ str       ┆           ┆           ┆   ┆           ┆ str       ┆ str       ┆          │\n",
            "│ μs, UTC]  ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
            "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
            "│ 2024-07-2 ┆ win10     ┆ fe80::c1a ┆ informati ┆ … ┆ C:\\Window ┆           ┆           ┆ Network  │\n",
            "│ 8 15:08:2 ┆           ┆ f:35de:60 ┆ on        ┆   ┆ s\\System3 ┆           ┆           ┆ connecti │\n",
            "│ 4.277 UTC ┆           ┆ 06:d4cf   ┆           ┆   ┆ 2\\svchost ┆           ┆           ┆ on detec │\n",
            "│           ┆           ┆           ┆           ┆   ┆ .exe      ┆           ┆           ┆ ted:     │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ Rul…     │\n",
            "│ 2024-07-2 ┆ win10     ┆ fe80::c1a ┆ informati ┆ … ┆ C:\\Window ┆           ┆           ┆ Network  │\n",
            "│ 8 15:08:2 ┆           ┆ f:35de:60 ┆ on        ┆   ┆ s\\System3 ┆           ┆           ┆ connecti │\n",
            "│ 4.488 UTC ┆           ┆ 06:d4cf   ┆           ┆   ┆ 2\\svchost ┆           ┆           ┆ on detec │\n",
            "│           ┆           ┆           ┆           ┆   ┆ .exe      ┆           ┆           ┆ ted:     │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ Rul…     │\n",
            "│ 2024-07-2 ┆ win10     ┆ fe80::c1a ┆ informati ┆ … ┆ C:\\Window ┆           ┆           ┆ Process  │\n",
            "│ 8 15:08:2 ┆           ┆ f:35de:60 ┆ on        ┆   ┆ s\\system3 ┆           ┆           ┆ accessed │\n",
            "│ 5.005 UTC ┆           ┆ 06:d4cf   ┆           ┆   ┆ 2\\svchost ┆           ┆           ┆ :        │\n",
            "│           ┆           ┆           ┆           ┆   ┆ .exe      ┆           ┆           ┆ RuleName │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ : -      │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ Ut…      │\n",
            "│ 2024-07-2 ┆ win10     ┆ fe80::c1a ┆ informati ┆ … ┆ C:\\Window ┆           ┆           ┆ Process  │\n",
            "│ 8 15:08:2 ┆           ┆ f:35de:60 ┆ on        ┆   ┆ s\\system3 ┆           ┆           ┆ accessed │\n",
            "│ 5.005 UTC ┆           ┆ 06:d4cf   ┆           ┆   ┆ 2\\svchost ┆           ┆           ┆ :        │\n",
            "│           ┆           ┆           ┆           ┆   ┆ .exe      ┆           ┆           ┆ RuleName │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ : -      │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ Ut…      │\n",
            "│ 2024-07-2 ┆ win10     ┆ fe80::c1a ┆ informati ┆ … ┆ C:\\Window ┆           ┆           ┆ Process  │\n",
            "│ 8 15:08:2 ┆           ┆ f:35de:60 ┆ on        ┆   ┆ s\\system3 ┆           ┆           ┆ accessed │\n",
            "│ 5.030 UTC ┆           ┆ 06:d4cf   ┆           ┆   ┆ 2\\svchost ┆           ┆           ┆ :        │\n",
            "│           ┆           ┆           ┆           ┆   ┆ .exe      ┆           ┆           ┆ RuleName │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ : -      │\n",
            "│           ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆ Ut…      │\n",
            "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘\n",
            "shape: (41, 2)\n",
            "┌─────────────────────┬───────┐\n",
            "│ parent_image        ┆ count │\n",
            "│ ---                 ┆ ---   │\n",
            "│ str                 ┆ u32   │\n",
            "╞═════════════════════╪═══════╡\n",
            "│ python.exe          ┆ 1044  │\n",
            "│ runonce.exe         ┆ 1     │\n",
            "│ Integrator.exe      ┆ 10    │\n",
            "│ CompatTelRunner.exe ┆ 2     │\n",
            "│ …                   ┆ …     │\n",
            "│ AvastUI.exe         ┆ 6     │\n",
            "│ AvLaunch.exe        ┆ 1     │\n",
            "│ upfc.exe            ┆ 1     │\n",
            "│ winlogon.exe        ┆ 8     │\n",
            "└─────────────────────┴───────┘\n",
            "shape: (91, 2)\n",
            "┌───────────────────────────────────┬───────┐\n",
            "│ target_filename                   ┆ count │\n",
            "│ ---                               ┆ ---   │\n",
            "│ str                               ┆ u32   │\n",
            "╞═══════════════════════════════════╪═══════╡\n",
            "│ C:\\Users\\student\\AppData\\Local\\M… ┆ 1     │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ 1     │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ 1     │\n",
            "│ C:\\Program Files (x86)\\Microsoft… ┆ 1     │\n",
            "│ …                                 ┆ …     │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ 1     │\n",
            "│ C:\\Program Files (x86)\\Microsoft… ┆ 1     │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ 1     │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ 1     │\n",
            "└───────────────────────────────────┴───────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Qualitative behaviour analysis: temporary folder on Windows 10\n",
        "\n",
        "It seems that suspicious activity involves temporary folders. A list of the temp folders is used to define a condition."
      ],
      "metadata": {
        "id": "IuK7hfb_mq8e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def is_temp_folder(path):\n",
        "    if not path:\n",
        "        return False\n",
        "\n",
        "    # Convert path to lowercase for case-insensitive comparison\n",
        "    lower_path = path.lower()\n",
        "\n",
        "    # Check if the path contains \"\\temp\\\" or ends with \"\\temp\"\n",
        "    if \"\\temp\\\\\" in lower_path or lower_path.endswith(\"\\\\temp\"):\n",
        "        return True\n",
        "\n",
        "    # Check for specific temp folder patterns\n",
        "    temp_patterns = [\n",
        "        r\"c:\\windows\\temp\",\n",
        "        r\"c:\\users\\*\\appdata\\local\\temp\",\n",
        "        r\"c:\\users\\*\\appdata\\locallow\\temp\",\n",
        "        r\"c:\\users\\*\\appdata\\roaming\\temp\",\n",
        "        r\"c:\\temp\",\n",
        "        r\"c:\\windows\\softwaredistriβution\\download\",\n",
        "    ]\n",
        "\n",
        "    for pattern in temp_patterns:\n",
        "        if pattern.startswith(r\"c:\\users\\*\"):\n",
        "            # Replace the wildcard with the actual username\n",
        "            user_pattern = pattern.replace(\"*\", path.split(\"\\\\\")[2])\n",
        "            if lower_path.startswith(user_pattern):\n",
        "                return True\n",
        "        elif lower_path.startswith(pattern):\n",
        "            return True\n",
        "\n",
        "    return False\n",
        "\n",
        "def get_filename(path):\n",
        "    return path.split(\"\\\\\")[-1] if path else \"\"\n",
        "\n",
        "# Add new columns to the DataFrame in a single operation\n",
        "df_f = df_f.with_columns([\n",
        "    pl.col(\"target_filename\").map_elements(lambda x: \"Yes\" if is_temp_folder(x) else \"No\").alias(\"temp_folder\"),\n",
        "    pl.col(\"target_filename\").map_elements(get_filename).alias(\"filename\")\n",
        "])\n",
        "\n",
        "# Print the first few rows where temp_folder is \"Yes\"\n",
        "print(df_f.filter(pl.col(\"temp_folder\") == \"Yes\").select([\"target_filename\", \"temp_folder\", \"filename\"]).head(10))\n",
        "\n",
        "# Print value counts for temp_folder column\n",
        "print(df_f[\"temp_folder\"].value_counts())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b0wLxKfzsg1e",
        "outputId": "83ec6ca7-1ef3-4486-a2b9-4cc1845b18e9"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (10, 3)\n",
            "┌───────────────────────────────────┬─────────────┬────────────────────────┐\n",
            "│ target_filename                   ┆ temp_folder ┆ filename               │\n",
            "│ ---                               ┆ ---         ┆ ---                    │\n",
            "│ str                               ┆ str         ┆ str                    │\n",
            "╞═══════════════════════════════════╪═════════════╪════════════════════════╡\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ file.exe               │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ cli-32.exe             │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ cli-64.exe             │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ cli-arm64.exe          │\n",
            "│ …                                 ┆ …           ┆ …                      │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ gui-64.exe             │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ gui-arm64.exe          │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ gui.exe                │\n",
            "│ C:\\Users\\student\\AppData\\Local\\T… ┆ Yes         ┆ wininst-10.0-amd64.exe │\n",
            "└───────────────────────────────────┴─────────────┴────────────────────────┘\n",
            "shape: (2, 2)\n",
            "┌─────────────┬───────┐\n",
            "│ temp_folder ┆ count │\n",
            "│ ---         ┆ ---   │\n",
            "│ str         ┆ u32   │\n",
            "╞═════════════╪═══════╡\n",
            "│ Yes         ┆ 166   │\n",
            "│ No          ┆ 13289 │\n",
            "└─────────────┴───────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Quantitative behaviour analysis: How many \"bad\" conditions did this yield?"
      ],
      "metadata": {
        "id": "mVu5858Xm8qU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "count = df_f.filter((pl.col(\"temp_folder\") == \"Yes\") & pl.col(\"filename\").str.contains(\"file.exe\")).height\n",
        "print(count)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uj7NV2wGPHi2",
        "outputId": "59ffa6bd-ce49-43ab-92e0-67e72adc21c7"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "114\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Store the data as CSV and JSON\n",
        "\n",
        "CSV and JSON are popular data format standards.\n",
        "\n",
        "The multiline CSV can be opened with `vim`, and searched. For the JSON this is harder, but there are command-line utils like `jq`. The analysis focuses on the CSV, but the JSON file is shared as well (https://github.com/norandom/log2ml/releases/tag/lab)"
      ],
      "metadata": {
        "id": "BGPg1RDmxMVE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_f.write_csv(\"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered.csv\", include_header=True)"
      ],
      "metadata": {
        "id": "ngGJUIsXu5fL"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_f.write_json(\"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered.json\")"
      ],
      "metadata": {
        "id": "lRQS5xLTvpZv"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Labeling based on Python\n",
        "\n",
        "Supervised ML Algorithms require labelled data to learn from.\n",
        "In the following the conditions are coded in Python to speed things up.\n",
        "\n",
        "The analysis can be conducted manually, and samples can be marked per index no."
      ],
      "metadata": {
        "id": "GgFNrU80xQsF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def define_label(row):\n",
        "    conditions = {\n",
        "        (\"EXCEL.EXE\" in row['image'] and \".exe\" in row['target_filename'] and row['temp_folder'] == \"Yes\"): \"bad\",\n",
        "        (row['parent_image'] == \"EXCEL.EXE\" and row['temp_folder'] == \"Yes\" and row['image'].lower().endswith('.exe')): \"bad\",\n",
        "        # Add more conditions here if needed\n",
        "    }\n",
        "    return conditions.get(True, \"good\")\n",
        "\n",
        "# Apply the label to the DataFrame\n",
        "df_f = df_f.with_columns(pl.struct(df_f.columns).map_elements(define_label).alias(\"label\"))\n",
        "\n",
        "# Print the first few rows where the label is \"bad\"\n",
        "print(df_f.filter(pl.col(\"label\") == \"bad\").select([\"image\", \"parent_image\", \"target_filename\", \"temp_folder\", \"label\"]).head(10))\n",
        "\n",
        "# Print value counts for the label column\n",
        "print(df_f[\"label\"].value_counts())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X5ptzPnfxP9M",
        "outputId": "0ac19338-925b-4d3e-c43f-baa98a138d82"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (10, 5)\n",
            "┌──────────────────────────────┬──────────────┬──────────────────────────────┬─────────────┬───────┐\n",
            "│ image                        ┆ parent_image ┆ target_filename              ┆ temp_folder ┆ label │\n",
            "│ ---                          ┆ ---          ┆ ---                          ┆ ---         ┆ ---   │\n",
            "│ str                          ┆ str          ┆ str                          ┆ str         ┆ str   │\n",
            "╞══════════════════════════════╪══════════════╪══════════════════════════════╪═════════════╪═══════╡\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ …                            ┆ …            ┆ …                            ┆ …           ┆ …     │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "│ C:\\Program Files\\Microsoft   ┆              ┆ C:\\Users\\student\\AppData\\Loc ┆ Yes         ┆ bad   │\n",
            "│ Offic…                       ┆              ┆ al\\T…                        ┆             ┆       │\n",
            "└──────────────────────────────┴──────────────┴──────────────────────────────┴─────────────┴───────┘\n",
            "shape: (2, 2)\n",
            "┌───────┬───────┐\n",
            "│ label ┆ count │\n",
            "│ ---   ┆ ---   │\n",
            "│ str   ┆ u32   │\n",
            "╞═══════╪═══════╡\n",
            "│ bad   ┆ 114   │\n",
            "│ good  ┆ 13341 │\n",
            "└───────┴───────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Check of the Adversary Emulation data: a simulation dataset for InfoSec\n",
        "\n",
        "The dataset contains security simulation (Adversary Emulation) data. A software robot opened 1000 files. file.exe was dropped to a temp folder."
      ],
      "metadata": {
        "id": "MES2a9BdnkU4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Filter the DataFrame for rows where label is \"bad\" and select specified columns\n",
        "bad_df = df_f.filter(pl.col(\"label\") == \"bad\").select([\n",
        "    \"image\", \"parent_image\", \"filename\", \"temp_folder\", \"label\"\n",
        "])\n",
        "\n",
        "# Write the filtered DataFrame to a CSV file\n",
        "bad_df.write_csv(\"bad.csv\")\n",
        "\n",
        "# Print the first 10 rows of the bad DataFrame\n",
        "print(bad_df.head(10))\n",
        "\n",
        "# Print the total count of \"bad\" rows\n",
        "total_bad_count = bad_df.shape[0]\n",
        "print(f\"\\nTotal number of 'bad' rows: {total_bad_count}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dRH3AReO_i5D",
        "outputId": "264a8f1e-5f67-4be4-88a5-a8962df60071"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (10, 5)\n",
            "┌───────────────────────────────────┬──────────────┬──────────┬─────────────┬───────┐\n",
            "│ image                             ┆ parent_image ┆ filename ┆ temp_folder ┆ label │\n",
            "│ ---                               ┆ ---          ┆ ---      ┆ ---         ┆ ---   │\n",
            "│ str                               ┆ str          ┆ str      ┆ str         ┆ str   │\n",
            "╞═══════════════════════════════════╪══════════════╪══════════╪═════════════╪═══════╡\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ …                                 ┆ …            ┆ …        ┆ …           ┆ …     │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "│ C:\\Program Files\\Microsoft Offic… ┆              ┆ file.exe ┆ Yes         ┆ bad   │\n",
            "└───────────────────────────────────┴──────────────┴──────────┴─────────────┴───────┘\n",
            "\n",
            "Total number of 'bad' rows: 114\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Vectorization with Linformer\n",
        "\n",
        "https://ai.meta.com/blog/how-facebook-uses-super-efficient-ai-models-to-detect-hate-speech/\n",
        "\n",
        "https://github.com/tatp22/linformer-pytorch\n",
        "\n",
        "Facebook uses Linformer to detect hate speech. In Social Networks hate speech isn't constructed like prosa literature, but contains slang and symbol characters as a means to obscure the meaning to outsiders. This indicates that Linformer model could potencially be useful for semi-natural language, which is also common for log- and telemetry data (in InfoSec9"
      ],
      "metadata": {
        "id": "9-1dkT2mQHvJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "_Vgi0rjn_RIn"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Linformer parameter 1:1 from the researchers\n",
        "# then the input_size is determined based on the max no. of tokens\n",
        "# the positional embedding flag is on by default\n",
        "\n",
        "from linformer_pytorch import LinformerLM\n",
        "import torch\n",
        "\n",
        "linformer_model = LinformerLM(\n",
        "        num_tokens=30000, # Number of tokens in the LM\n",
        "        input_size=700, # Dimension 1 of the input\n",
        "        channels=64, # Dimension 2 of the input\n",
        "        dim_d=None, # Overwrites the inner dim of the attention heads. If None, sticks with the recommended channels // nhead, as in the \"Attention is all you need\" paper\n",
        "        dim_k=128, # The second dimension of the P_bar matrix from the paper\n",
        "        dim_ff=128, # Dimension in the feed forward network\n",
        "        dropout_ff=0.15, # Dropout for feed forward network\n",
        "        nhead=4, # Number of attention heads\n",
        "        depth=2, # How many times to run the model\n",
        "        dropout=0.1, # How much dropout to apply to P_bar after softmax\n",
        "        activation=\"gelu\", # What activation to use. Currently, only gelu and relu supported, and only on ff network.\n",
        "        checkpoint_level=\"C0\", # What checkpoint level to use. For more information, see below.\n",
        "        parameter_sharing=\"layerwise\", # What level of parameter sharing to use. For more information, see below.\n",
        "        k_reduce_by_layer=0, # Going down `depth`, how much to reduce `dim_k` by, for the `E` and `F` matrices. Will have a minimum value of 1.\n",
        "        full_attention=False, # Use full attention instead, for O(n^2) time and space complexity. Included here just for comparison\n",
        "        include_ff=True, # Whether or not to include the Feed Forward layer\n",
        "        w_o_intermediate_dim=None, # If not None, have 2 w_o matrices, such that instead of `dim*nead,channels`, you have `dim*nhead,w_o_int`, and `w_o_int,channels`\n",
        "        emb_dim=128, # If you want the embedding dimension to be different than the channels for the Linformer\n",
        "        causal=False, # If you want this to be a causal Linformer, where the upper right of the P_bar matrix is masked out.\n",
        "        method=\"learnable\", # The method of how to perform the projection. Supported methods are 'convolution', 'learnable', and 'no_params'\n",
        "        ff_intermediate=None, # See the section below for more information\n",
        "        ).cuda()"
      ],
      "metadata": {
        "id": "SsW6vlQNQhAy"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "## The model can get fine-tuned. This isn't necessary, but it's possible for optimization.\n",
        "## Many ML problems become optimization problems, once the models start to perform.\n",
        "## AutoML (with GP) can accelerate the optimization, especially when it comes to balancing\n",
        "## performance and precision."
      ],
      "metadata": {
        "id": "Q-Y1HQMqzN-5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Example messages from the two classes (for a single-label classifier)\n",
        "\n",
        "We have two classes:\n",
        "\n",
        "\n",
        "1. good\n",
        "2. bad\n",
        "\n",
        "\n",
        "\n",
        "The bad-class is a minority class. This is typical for domains such as InfoSec (or medicine etc.) where the identification of an abnormality needs to be performed. For ML handling minority class problems poses an extra challenge."
      ],
      "metadata": {
        "id": "91khWgQT3DVa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(df_f.columns)\n",
        "print(df_f[[\"label\", \"filtered_message\"]])\n",
        "\n",
        "import polars as pl\n",
        "import random\n",
        "\n",
        "bad_messages = df_f.filter(pl.col('label') == 'bad').select('filtered_message')\n",
        "good_messages = df_f.filter(pl.col('label') == 'good').select('filtered_message')\n",
        "\n",
        "random_bad_message = \"\"\n",
        "random_good_message = \"\"\n",
        "\n",
        "if bad_messages.height > 0:\n",
        "    # Get a random index\n",
        "    random_index = random.randint(0, bad_messages.height - 1)\n",
        "\n",
        "    # Select the message at that random index\n",
        "    random_bad_message = bad_messages.row(random_index)[0]\n",
        "    print(random_bad_message)\n",
        "else:\n",
        "    print(\"No bad messages labeled found.\")\n",
        "\n",
        "print(\"\\n\\n\")\n",
        "\n",
        "if good_messages.height > 0:\n",
        "    # Get a random index\n",
        "    random_index = random.randint(0, good_messages.height - 1)\n",
        "\n",
        "    # Select the message at that random index\n",
        "    random_good_message = good_messages.row(random_index)[0]\n",
        "    print(random_good_message)\n",
        "else:\n",
        "    print(\"No good messages labeled found.\")"
      ],
      "metadata": {
        "id": "4SguqPPKSdpJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16c10c29-88d4-4639-f645-1480bab8a94c"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['@timestamp', 'host.hostname', 'host.ip', 'log.level', 'winlog.event_id', 'winlog.task', 'message', 'filtered_message', 'image', 'target_filename', 'parent_image', 'text', 'temp_folder', 'filename', 'label']\n",
            "shape: (13_455, 2)\n",
            "┌───────┬──────────────────────────────┐\n",
            "│ label ┆ filtered_message             │\n",
            "│ ---   ┆ ---                          │\n",
            "│ str   ┆ str                          │\n",
            "╞═══════╪══════════════════════════════╡\n",
            "│ good  ┆ Network connection detected: │\n",
            "│       ┆ Rul…                         │\n",
            "│ good  ┆ Network connection detected: │\n",
            "│       ┆ Rul…                         │\n",
            "│ good  ┆ Process accessed:            │\n",
            "│       ┆ RuleName: -                  │\n",
            "│       ┆ Ut…                          │\n",
            "│ good  ┆ Process accessed:            │\n",
            "│       ┆ RuleName: -                  │\n",
            "│       ┆ Ut…                          │\n",
            "│ …     ┆ …                            │\n",
            "│ good  ┆ Process accessed:            │\n",
            "│       ┆ RuleName: -                  │\n",
            "│       ┆ Ut…                          │\n",
            "│ good  ┆ Process accessed:            │\n",
            "│       ┆ RuleName: -                  │\n",
            "│       ┆ Ut…                          │\n",
            "│ good  ┆ Process accessed:            │\n",
            "│       ┆ RuleName: -                  │\n",
            "│       ┆ Ut…                          │\n",
            "│ good  ┆ Process Create:              │\n",
            "│       ┆ RuleName: -                  │\n",
            "│       ┆ UtcT…                        │\n",
            "└───────┴──────────────────────────────┘\n",
            "File created:\n",
            "RuleName: EXE\n",
            "UtcTime: 2024-07-28 20:40:26.601\n",
            "ProcessGuid: {18e8265a-acb5-66a6-fb0d-000000004400}\n",
            "ProcessId: 2428\n",
            "Image: C:\\Program Files\\Microsoft Office\\Root\\Office16\\EXCEL.EXE\n",
            "TargetFilename: C:\\Users\\student\\AppData\\Local\\Temp\\file.exe\n",
            "CreationUtcTime: 2024-07-23 14:24:50.520\n",
            "\n",
            "\n",
            "\n",
            "File created:\n",
            "RuleName: DLL\n",
            "UtcTime: 2024-07-28 15:23:00.514\n",
            "ProcessGuid: {18e8265a-6254-66a6-e102-000000004400}\n",
            "ProcessId: 10948\n",
            "Image: C:\\Program Files\\Avast Software\\Avast\\Setup\\Instup.exe\n",
            "TargetFilename: C:\\Program Files\\Avast Software\\Avast\\setup\\uat64.dll\n",
            "CreationUtcTime: 2023-03-09 19:28:07.792\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Example messages from the two classes (for a single-label classifier) (pandas framework code)\n",
        "\n",
        "This is the same but with Pandas.\n",
        "\n",
        "The variables are the same:\n",
        "\n",
        "\n",
        "```\n",
        "random_bad_message = \"\"\n",
        "random_good_message = \"\"\n",
        "```\n",
        "\n",
        "The following code is needed if the data get downloaded from the release and not processed step by step."
      ],
      "metadata": {
        "id": "ZY-8c7Q_3mEg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import random\n",
        "\n",
        "# Assuming df_f is your pandas DataFrame\n",
        "print(df_f.columns)\n",
        "print(df_f[[\"label\", \"filtered_message\"]])\n",
        "\n",
        "bad_messages = df_f[df_f['label'] == 'bad']['filtered_message']\n",
        "good_messages = df_f[df_f['label'] == 'good']['filtered_message']\n",
        "\n",
        "random_bad_message = \"\"\n",
        "random_good_message = \"\"\n",
        "\n",
        "if not bad_messages.empty:\n",
        "    random_bad_message = bad_messages.sample().iloc[0]\n",
        "    print(random_bad_message)\n",
        "else:\n",
        "    print(\"No bad messages labeled found.\")\n",
        "\n",
        "print(\"\\n\\n\")\n",
        "\n",
        "if not good_messages.empty:\n",
        "    random_good_message = good_messages.sample().iloc[0]\n",
        "    print(random_good_message)\n",
        "else:\n",
        "    print(\"No good messages labeled found.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cVZe3nmRBwc2",
        "outputId": "dd316cbf-c3c3-44fd-9614-497382fc95b2"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Index(['@timestamp', 'host.hostname', 'host.ip', 'log.level',\n",
            "       'winlog.event_id', 'winlog.task', 'message', 'filtered_message',\n",
            "       'image', 'target_filename', 'parent_image', 'text', 'temp_folder',\n",
            "       'filename', 'label', 'message_vector'],\n",
            "      dtype='object')\n",
            "      label                                   filtered_message\n",
            "0      good  Network connection detected:\\nRuleName: -\\nUtc...\n",
            "1      good  Network connection detected:\\nRuleName: -\\nUtc...\n",
            "2      good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "3      good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "4      good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "...     ...                                                ...\n",
            "13450  good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "13451  good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "13452  good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "13453  good  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...\n",
            "13454  good  Process Create:\\nRuleName: -\\nUtcTime: 2024-07...\n",
            "\n",
            "[13455 rows x 2 columns]\n",
            "File created:\n",
            "RuleName: EXE\n",
            "UtcTime: 2024-07-28 15:53:22.557\n",
            "ProcessGuid: {18e8265a-696d-66a6-7705-000000004400}\n",
            "ProcessId: 10624\n",
            "Image: C:\\Program Files\\Microsoft Office\\Root\\Office16\\EXCEL.EXE\n",
            "TargetFilename: C:\\Users\\student\\AppData\\Local\\Temp\\file.exe\n",
            "CreationUtcTime: 2024-07-23 14:24:50.520\n",
            "\n",
            "\n",
            "\n",
            "Dns query:\n",
            "RuleName: -\n",
            "UtcTime: 2024-07-28 19:08:06.591\n",
            "ProcessGuid: {18e8265a-9715-66a6-600b-000000004400}\n",
            "ProcessId: 540\n",
            "QueryName: messaging\n",
            "QueryStatus: 0\n",
            "QueryResults: type:  5 prod-campaignaggregator.omexexternallfb.office.net.akadns.net;::ffff:52.109.16.3;\n",
            "Image: C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Vectorize text column in the DataFrame"
      ],
      "metadata": {
        "id": "h4cuBLWrSNVs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tokenizers import Tokenizer\n",
        "import torch\n",
        "import numpy as np\n",
        "import polars as pl\n",
        "\n",
        "# Load the custom tokenizer\n",
        "tokenizer = Tokenizer.from_file(\"log_tokenizer.json\")\n",
        "\n",
        "# Define the device (assuming you're using PyTorch and want to specify CPU or GPU)\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "print(\"This uses a \" + str(device) + \" device\")\n",
        "\n",
        "def vectorize_text(text):\n",
        "    MAX_LENGTH = 700  # Define the maximum length of tokens for the model\n",
        "\n",
        "    # Tokenize using the custom tokenizer\n",
        "    encoded = tokenizer.encode(text)\n",
        "\n",
        "    # Get token IDs\n",
        "    input_ids = encoded.ids\n",
        "\n",
        "    # Ensure the input_ids length is exactly MAX_LENGTH\n",
        "    input_ids = input_ids[:MAX_LENGTH] if len(input_ids) > MAX_LENGTH else input_ids + [0] * (MAX_LENGTH - len(input_ids))\n",
        "\n",
        "    # Convert to PyTorch tensor and move to the appropriate device\n",
        "    input_ids = torch.tensor([input_ids], dtype=torch.long).to(device)\n",
        "\n",
        "    # Get the model outputs, ensuring the input tensor is the correct size\n",
        "    outputs = linformer_model(input_ids)  # Now passing the tensor directly\n",
        "\n",
        "    # Assuming outputs is the tensor of interest\n",
        "    vector = outputs.mean(dim=1).detach()  # Detach the tensor from the GPU\n",
        "    return vector.cpu().numpy()  # Move tensor back to CPU and convert to numpy"
      ],
      "metadata": {
        "id": "EyvdFj83SQKI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "882141d8-1956-4f6b-ba33-b5dbe205e4af"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This uses a cuda device\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# The following code performs the high-dimensional vector projection with the Linformer model\n",
        "\n",
        "This can take some minutes on a GPU system.\n",
        "On a CPU system the runtime is exponentially higher."
      ],
      "metadata": {
        "id": "kSVB7YH-4czH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "# Assuming df_f is a Polars DataFrame with a column \"filtered_message\"\n",
        "df_f = df_f.with_columns(\n",
        "    pl.col(\"filtered_message\").map_elements(lambda x: vectorize_text(x).flatten(), return_dtype=pl.Object).alias(\"message_vector\")\n",
        ")"
      ],
      "metadata": {
        "id": "doS16Nq4S37g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4e11e7b6-60f1-4caa-e268-b2f64e1a5852"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 2min 4s, sys: 1.05 s, total: 2min 5s\n",
            "Wall time: 2min 5s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Saving the vectorized dataset as Parquet\n",
        "\n",
        "This may require up to 25 GB RAM and 4 GB disk space. The vector data is very sparse (30k dims). That's not strictly necessary, but since we are going to reduce this with a PCA anyways it's not a big development challenge. The \"Curse of Dimensionality\" gets llustrated this way. Other models, such as LongFormer, produce less dimensions and may be more suitable form a development standpoint.\n",
        "\n",
        "The code uses a mixture of Pandas and Polars, because Polars isn't feature-complete."
      ],
      "metadata": {
        "id": "ScKF9UeB66LK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import polars as pl\n",
        "import numpy as np\n",
        "import json\n",
        "\n",
        "# Function to convert vector to string\n",
        "def vector_to_string(vec):\n",
        "    return json.dumps(vec.tolist())\n",
        "\n",
        "# Function to convert string back to vector\n",
        "def string_to_vector(s):\n",
        "    return np.array(json.loads(s))\n",
        "\n",
        "# Convert vector column to string representation\n",
        "df_f = df_f.with_columns(\n",
        "    pl.col(\"message_vector\").map_elements(vector_to_string).alias(\"message_vector_str\")\n",
        ")\n",
        "\n",
        "# Drop the original vector column\n",
        "df_f = df_f.drop(\"message_vector\")\n",
        "\n",
        "# Write to Parquet\n",
        "df_f.write_parquet(\"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered_with_vectors.parquet\")"
      ],
      "metadata": {
        "id": "clEaIJjHXRhM"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(df_f.columns)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iixa8gHy80B6",
        "outputId": "d38be1e4-0381-41e2-a553-a7a8a9b203d8"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['@timestamp', 'host.hostname', 'host.ip', 'log.level', 'winlog.event_id', 'winlog.task', 'message', 'filtered_message', 'image', 'target_filename', 'parent_image', 'text', 'temp_folder', 'filename', 'label', 'message_vector_str']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Import the data\n",
        "\n",
        "The vectorized file can't get uploaded as a GitHub release, because it's too big. As a side-objective the data has been shared to the Kaggle community:\n",
        "\n",
        "https://www.kaggle.com/datasets/mariusciepluch/log2ml-blindtest-maldoc-activity-capture"
      ],
      "metadata": {
        "id": "7LTDH-8RecLy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Kaggle download: log2ml dataset with Adversary Emulation data\n",
        "\n",
        "The following code download the data using the kaggle util. Authentication isn't necessary, because this is a public dataset."
      ],
      "metadata": {
        "id": "zmJzQP08sFfd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import subprocess\n",
        "\n",
        "# Set Kaggle config directory to current working directory\n",
        "os.environ['KAGGLE_CONFIG_DIR'] = os.getcwd()\n",
        "\n",
        "# Construct the Kaggle CLI command\n",
        "command = \"kaggle datasets download -d mariusciepluch/log2ml-blindtest-maldoc-activity-capture --unzip\"\n",
        "\n",
        "# Run the command\n",
        "print(\"Downloading dataset...\")\n",
        "result = subprocess.run(command, shell=True, capture_output=True, text=True)\n",
        "\n",
        "# Check if the command was successful\n",
        "if result.returncode == 0:\n",
        "    print(\"Dataset downloaded successfully!\")\n",
        "else:\n",
        "    print(\"Error downloading dataset:\")\n",
        "    print(result.stderr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FWa0Jq8ocMpJ",
        "outputId": "96b34f8f-0541-498f-97eb-057883e52333"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading dataset...\n",
            "Dataset downloaded successfully!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load the vector data into polars (with pandas)\n",
        "\n",
        "Tpot uses Pandas. Therefore the data needs to be in a Pandas DataFrame. It's important to note that we want the vector dimensions to remain the same. It should be 30k. Serializing high-dimensional vector-data to parquet has been a development challenge."
      ],
      "metadata": {
        "id": "g7Tzv9jXsLWs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "import polars as pl\n",
        "\n",
        "# Read from Parquet using pandas\n",
        "pdf_read = pd.read_parquet(\"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered_with_vectors.parquet\")\n",
        "\n",
        "# Function to convert JSON-encoded strings back to numpy arrays\n",
        "def string_to_vector(s):\n",
        "    return np.array(json.loads(s))\n",
        "\n",
        "# Convert JSON strings back to numpy arrays\n",
        "pdf_read['message_vector'] = pdf_read['message_vector_str'].apply(string_to_vector)\n",
        "\n",
        "# Verify the shape of the vector\n",
        "print(pdf_read['message_vector'].apply(len).head())\n",
        "\n",
        "# Check a sample vector to ensure dimensionality is preserved\n",
        "sample_vector = pdf_read['message_vector'].iloc[0]\n",
        "print(f\"Sample vector shape: {sample_vector.shape}\")\n",
        "\n",
        "# Convert back to Polars and drop the string column\n",
        "df_read = pl.from_pandas(pdf_read.drop(columns='message_vector_str'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "981FBJ_rdh-V",
        "outputId": "e370d4e9-078f-4276-ee6b-3aed7a33344e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0    30000\n",
            "1    30000\n",
            "2    30000\n",
            "3    30000\n",
            "4    30000\n",
            "Name: message_vector, dtype: int64\n",
            "Sample vector shape: (30000,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Verify in Polars\n",
        "print(df_read.select([pl.col(\"message_vector\").list.len().alias(\"vector_lengths\")]).head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZZ1CPkp0AL-z",
        "outputId": "897f19f7-37ab-48a5-b36a-f55ee2c26d4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape: (5, 1)\n",
            "┌────────────────┐\n",
            "│ vector_lengths │\n",
            "│ ---            │\n",
            "│ u32            │\n",
            "╞════════════════╡\n",
            "│ 30000          │\n",
            "│ 30000          │\n",
            "│ 30000          │\n",
            "│ 30000          │\n",
            "│ 30000          │\n",
            "└────────────────┘\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load the vector data into pandas (direct)"
      ],
      "metadata": {
        "id": "_LydloD7sczq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "\n",
        "# Read from Parquet using pandas\n",
        "pdf_read = pd.read_parquet(\"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered_with_vectors.parquet\")\n",
        "\n",
        "# Function to convert JSON-encoded strings back to numpy arrays\n",
        "def string_to_vector(s):\n",
        "    return np.array(json.loads(s))\n",
        "\n",
        "# Convert JSON strings back to numpy arrays\n",
        "pdf_read['message_vector'] = pdf_read['message_vector_str'].apply(string_to_vector)\n",
        "\n",
        "# Verify the shape of the vector\n",
        "print(\"Vector lengths:\")\n",
        "print(pdf_read['message_vector'].apply(len).head())\n",
        "\n",
        "# Check a sample vector to ensure dimensionality is preserved\n",
        "sample_vector = pdf_read['message_vector'].iloc[0]\n",
        "print(f\"\\nSample vector shape: {sample_vector.shape}\")\n",
        "\n",
        "# Drop the string column as it's no longer needed\n",
        "pdf_read = pdf_read.drop(columns='message_vector_str')\n",
        "\n",
        "# Verify vector lengths (equivalent to the Polars operation)\n",
        "print(\"\\nVector lengths (pandas equivalent of Polars operation):\")\n",
        "print(pdf_read['message_vector'].apply(len).head())\n",
        "\n",
        "# If you need to see the full DataFrame structure\n",
        "print(\"\\nDataFrame info:\")\n",
        "pdf_read.info()\n",
        "\n",
        "# If you want to see the first few rows of the DataFrame\n",
        "print(\"\\nFirst few rows of the DataFrame:\")\n",
        "print(pdf_read.head())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ePLz8udYDHsK",
        "outputId": "b5a5f575-8104-4887-d3c4-601551634b0d"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Vector lengths:\n",
            "0    30000\n",
            "1    30000\n",
            "2    30000\n",
            "3    30000\n",
            "4    30000\n",
            "Name: message_vector, dtype: int64\n",
            "\n",
            "Sample vector shape: (30000,)\n",
            "\n",
            "Vector lengths (pandas equivalent of Polars operation):\n",
            "0    30000\n",
            "1    30000\n",
            "2    30000\n",
            "3    30000\n",
            "4    30000\n",
            "Name: message_vector, dtype: int64\n",
            "\n",
            "DataFrame info:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 13455 entries, 0 to 13454\n",
            "Data columns (total 16 columns):\n",
            " #   Column            Non-Null Count  Dtype              \n",
            "---  ------            --------------  -----              \n",
            " 0   @timestamp        13455 non-null  datetime64[us, UTC]\n",
            " 1   host.hostname     13455 non-null  object             \n",
            " 2   host.ip           13455 non-null  object             \n",
            " 3   log.level         13455 non-null  object             \n",
            " 4   winlog.event_id   13455 non-null  int64              \n",
            " 5   winlog.task       13455 non-null  object             \n",
            " 6   message           13455 non-null  object             \n",
            " 7   filtered_message  13455 non-null  object             \n",
            " 8   image             13455 non-null  object             \n",
            " 9   target_filename   13455 non-null  object             \n",
            " 10  parent_image      13455 non-null  object             \n",
            " 11  text              13455 non-null  object             \n",
            " 12  temp_folder       13455 non-null  object             \n",
            " 13  filename          13455 non-null  object             \n",
            " 14  label             13455 non-null  object             \n",
            " 15  message_vector    13455 non-null  object             \n",
            "dtypes: datetime64[us, UTC](1), int64(1), object(14)\n",
            "memory usage: 1.6+ MB\n",
            "\n",
            "First few rows of the DataFrame:\n",
            "                        @timestamp host.hostname                    host.ip  \\\n",
            "0 2024-07-28 15:08:24.277000+00:00         win10  fe80::c1af:35de:6006:d4cf   \n",
            "1 2024-07-28 15:08:24.488000+00:00         win10  fe80::c1af:35de:6006:d4cf   \n",
            "2 2024-07-28 15:08:25.005000+00:00         win10  fe80::c1af:35de:6006:d4cf   \n",
            "3 2024-07-28 15:08:25.005000+00:00         win10  fe80::c1af:35de:6006:d4cf   \n",
            "4 2024-07-28 15:08:25.030000+00:00         win10  fe80::c1af:35de:6006:d4cf   \n",
            "\n",
            "     log.level  winlog.event_id  \\\n",
            "0  information                3   \n",
            "1  information                3   \n",
            "2  information               10   \n",
            "3  information               10   \n",
            "4  information               10   \n",
            "\n",
            "                                         winlog.task  \\\n",
            "0  Network connection detected (rule: NetworkConn...   \n",
            "1  Network connection detected (rule: NetworkConn...   \n",
            "2             Process accessed (rule: ProcessAccess)   \n",
            "3             Process accessed (rule: ProcessAccess)   \n",
            "4             Process accessed (rule: ProcessAccess)   \n",
            "\n",
            "                                             message  \\\n",
            "0  Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "1  Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "2  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "3  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "4  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "\n",
            "                                    filtered_message  \\\n",
            "0  Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "1  Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "2  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "3  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "4  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "\n",
            "                             image target_filename parent_image  \\\n",
            "0  C:\\Windows\\System32\\svchost.exe                                \n",
            "1  C:\\Windows\\System32\\svchost.exe                                \n",
            "2  C:\\Windows\\system32\\svchost.exe                                \n",
            "3  C:\\Windows\\system32\\svchost.exe                                \n",
            "4  C:\\Windows\\system32\\svchost.exe                                \n",
            "\n",
            "                                                text temp_folder filename  \\\n",
            "0  Network connection detected:\\nRuleName: -\\nUtc...          No            \n",
            "1  Network connection detected:\\nRuleName: -\\nUtc...          No            \n",
            "2  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "3  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "4  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "\n",
            "  label                                     message_vector  \n",
            "0  good  [-0.2214650958776474, 0.04250260442495346, 0.2...  \n",
            "1  good  [-0.19032716751098633, 0.010175472125411034, 0...  \n",
            "2  good  [-0.26184871792793274, 0.08533265441656113, 0....  \n",
            "3  good  [-0.25396400690078735, 0.10707883536815643, 0....  \n",
            "4  good  [-0.24748775362968445, 0.06198200583457947, 0....  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import polars as pl\n",
        "import numpy as np\n",
        "\n",
        "df_f = pdf_read\n",
        "\n",
        "print(df_f)\n",
        "\n",
        "print()\n",
        "# Convert the 'message_vector' column to a NumPy array\n",
        "X = np.array(df_f['message_vector'].to_list())\n",
        "print(\"Original data shape:\", X.shape)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W849gxLgM3vP",
        "outputId": "ea78ee8e-1ef6-4e98-87f1-e5ffcfa11566"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                            @timestamp host.hostname  \\\n",
            "0     2024-07-28 15:08:24.277000+00:00         win10   \n",
            "1     2024-07-28 15:08:24.488000+00:00         win10   \n",
            "2     2024-07-28 15:08:25.005000+00:00         win10   \n",
            "3     2024-07-28 15:08:25.005000+00:00         win10   \n",
            "4     2024-07-28 15:08:25.030000+00:00         win10   \n",
            "...                                ...           ...   \n",
            "13450 2024-07-28 23:35:53.054000+00:00         win10   \n",
            "13451 2024-07-28 23:35:53.054000+00:00         win10   \n",
            "13452 2024-07-28 23:35:54.133000+00:00         win10   \n",
            "13453 2024-07-28 23:35:54.133000+00:00         win10   \n",
            "13454 2024-07-28 23:41:55.301000+00:00         win10   \n",
            "\n",
            "                         host.ip    log.level  winlog.event_id  \\\n",
            "0      fe80::c1af:35de:6006:d4cf  information                3   \n",
            "1      fe80::c1af:35de:6006:d4cf  information                3   \n",
            "2      fe80::c1af:35de:6006:d4cf  information               10   \n",
            "3      fe80::c1af:35de:6006:d4cf  information               10   \n",
            "4      fe80::c1af:35de:6006:d4cf  information               10   \n",
            "...                          ...          ...              ...   \n",
            "13450  fe80::c1af:35de:6006:d4cf  information               10   \n",
            "13451  fe80::c1af:35de:6006:d4cf  information               10   \n",
            "13452  fe80::c1af:35de:6006:d4cf  information               10   \n",
            "13453  fe80::c1af:35de:6006:d4cf  information               10   \n",
            "13454  fe80::c1af:35de:6006:d4cf  information                1   \n",
            "\n",
            "                                             winlog.task  \\\n",
            "0      Network connection detected (rule: NetworkConn...   \n",
            "1      Network connection detected (rule: NetworkConn...   \n",
            "2                 Process accessed (rule: ProcessAccess)   \n",
            "3                 Process accessed (rule: ProcessAccess)   \n",
            "4                 Process accessed (rule: ProcessAccess)   \n",
            "...                                                  ...   \n",
            "13450             Process accessed (rule: ProcessAccess)   \n",
            "13451             Process accessed (rule: ProcessAccess)   \n",
            "13452             Process accessed (rule: ProcessAccess)   \n",
            "13453             Process accessed (rule: ProcessAccess)   \n",
            "13454               Process Create (rule: ProcessCreate)   \n",
            "\n",
            "                                                 message  \\\n",
            "0      Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "1      Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "2      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "3      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "4      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "...                                                  ...   \n",
            "13450  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13451  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13452  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13453  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13454  Process Create:\\nRuleName: -\\nUtcTime: 2024-07...   \n",
            "\n",
            "                                        filtered_message  \\\n",
            "0      Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "1      Network connection detected:\\nRuleName: -\\nUtc...   \n",
            "2      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "3      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "4      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "...                                                  ...   \n",
            "13450  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13451  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13452  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13453  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...   \n",
            "13454  Process Create:\\nRuleName: -\\nUtcTime: 2024-07...   \n",
            "\n",
            "                                 image target_filename  parent_image  \\\n",
            "0      C:\\Windows\\System32\\svchost.exe                                 \n",
            "1      C:\\Windows\\System32\\svchost.exe                                 \n",
            "2      C:\\Windows\\system32\\svchost.exe                                 \n",
            "3      C:\\Windows\\system32\\svchost.exe                                 \n",
            "4      C:\\Windows\\system32\\svchost.exe                                 \n",
            "...                                ...             ...           ...   \n",
            "13450  C:\\Windows\\system32\\svchost.exe                                 \n",
            "13451  C:\\Windows\\system32\\svchost.exe                                 \n",
            "13452  C:\\Windows\\system32\\svchost.exe                                 \n",
            "13453  C:\\Windows\\system32\\svchost.exe                                 \n",
            "13454  C:\\Windows\\System32\\svchost.exe                  services.exe   \n",
            "\n",
            "                                                    text temp_folder filename  \\\n",
            "0      Network connection detected:\\nRuleName: -\\nUtc...          No            \n",
            "1      Network connection detected:\\nRuleName: -\\nUtc...          No            \n",
            "2      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "3      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "4      Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "...                                                  ...         ...      ...   \n",
            "13450  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "13451  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "13452  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "13453  Process accessed:\\nRuleName: -\\nUtcTime: 2024-...          No            \n",
            "13454  Process Create:\\nRuleName: -\\nUtcTime: 2024-07...          No            \n",
            "\n",
            "      label                                     message_vector  \n",
            "0      good  [-0.2214650958776474, 0.04250260442495346, 0.2...  \n",
            "1      good  [-0.19032716751098633, 0.010175472125411034, 0...  \n",
            "2      good  [-0.26184871792793274, 0.08533265441656113, 0....  \n",
            "3      good  [-0.25396400690078735, 0.10707883536815643, 0....  \n",
            "4      good  [-0.24748775362968445, 0.06198200583457947, 0....  \n",
            "...     ...                                                ...  \n",
            "13450  good  [-0.24104158580303192, 0.07091948390007019, 0....  \n",
            "13451  good  [-0.23055040836334229, 0.07124319672584534, 0....  \n",
            "13452  good  [-0.2496212273836136, 0.0811350867152214, 0.21...  \n",
            "13453  good  [-0.2501278519630432, 0.0767292007803917, 0.21...  \n",
            "13454  good  [-0.22038744390010834, 0.07044447958469391, 0....  \n",
            "\n",
            "[13455 rows x 16 columns]\n",
            "\n",
            "Original data shape: (13455, 30000)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import polars as pl\n",
        "import pandas as pd\n",
        "\n",
        "# Assuming df_f is your Polars DataFrame\n",
        "try:\n",
        "    # Extract data from Polars DataFrame columns\n",
        "    data = {col: df_f[col].to_list() for col in df_f.columns}\n",
        "    # Create a Pandas DataFrame from the extracted data\n",
        "    df = pd.DataFrame(data)\n",
        "except Exception as e:\n",
        "    print(f\"An error occurred: {e}\")\n",
        "\n",
        "# Display the Pandas DataFrame info and dtypes\n",
        "if 'df' in locals():\n",
        "    print(\"Pandas DataFrame:\")\n",
        "    print(df.info())\n",
        "    print(df.dtypes)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9JjYJzacaD-T",
        "outputId": "2c9214a3-b9b6-4ce6-fe6b-c88a2ea61550"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pandas DataFrame:\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 13455 entries, 0 to 13454\n",
            "Data columns (total 16 columns):\n",
            " #   Column            Non-Null Count  Dtype              \n",
            "---  ------            --------------  -----              \n",
            " 0   @timestamp        13455 non-null  datetime64[ns, UTC]\n",
            " 1   host.hostname     13455 non-null  object             \n",
            " 2   host.ip           13455 non-null  object             \n",
            " 3   log.level         13455 non-null  object             \n",
            " 4   winlog.event_id   13455 non-null  int64              \n",
            " 5   winlog.task       13455 non-null  object             \n",
            " 6   message           13455 non-null  object             \n",
            " 7   filtered_message  13455 non-null  object             \n",
            " 8   image             13455 non-null  object             \n",
            " 9   target_filename   13455 non-null  object             \n",
            " 10  parent_image      13455 non-null  object             \n",
            " 11  text              13455 non-null  object             \n",
            " 12  temp_folder       13455 non-null  object             \n",
            " 13  filename          13455 non-null  object             \n",
            " 14  label             13455 non-null  object             \n",
            " 15  message_vector    13455 non-null  object             \n",
            "dtypes: datetime64[ns, UTC](1), int64(1), object(14)\n",
            "memory usage: 1.6+ MB\n",
            "None\n",
            "@timestamp          datetime64[ns, UTC]\n",
            "host.hostname                    object\n",
            "host.ip                          object\n",
            "log.level                        object\n",
            "winlog.event_id                   int64\n",
            "winlog.task                      object\n",
            "message                          object\n",
            "filtered_message                 object\n",
            "image                            object\n",
            "target_filename                  object\n",
            "parent_image                     object\n",
            "text                             object\n",
            "temp_folder                      object\n",
            "filename                         object\n",
            "label                            object\n",
            "message_vector                   object\n",
            "dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# AutoML with TPOT (Supervised Learning)"
      ],
      "metadata": {
        "id": "ZqGIRBU6aE3U"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sparse vector data\n",
        "\n",
        "Check of the vector dimensions."
      ],
      "metadata": {
        "id": "0Xxcb4ITMAH_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X = np.array(df['message_vector'].tolist())\n",
        "print(X.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5NHDYuDAiUis",
        "outputId": "11f23f11-961f-4a2f-a0fb-810f270c9df0"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(13455, 30000)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# TPOT for Supervised Learning with Algo stats"
      ],
      "metadata": {
        "id": "Gf4Fu4cDs4pr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder, QuantileTransformer\n",
        "from sklearn.decomposition import PCA\n",
        "from tpot import TPOTClassifier\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.pipeline import Pipeline\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from collections import defaultdict\n",
        "import time\n",
        "import re\n",
        "import gc\n",
        "\n",
        "# Assuming df is already loaded and contains 'message_vector' and 'label' columns\n",
        "\n",
        "# Encode labels\n",
        "le = LabelEncoder()\n",
        "df['label_encoded'] = le.fit_transform(df['label'])\n",
        "\n",
        "# Split data\n",
        "X = np.array(df['message_vector'].tolist())\n",
        "y = df['label_encoded'].values\n",
        "\n",
        "# Initialize results storage\n",
        "results = defaultdict(list)\n",
        "\n",
        "# Number of runs\n",
        "n_runs = 3\n",
        "\n",
        "# Function to extract number of features selected\n",
        "def get_n_features(pipeline_str):\n",
        "    match = re.search(r'SelectPercentile\\(score_func=f_classif, percentile=(\\d+)\\)', pipeline_str)\n",
        "    if match:\n",
        "        percentile = int(match.group(1))\n",
        "        return int(n_components * percentile / 100)\n",
        "    return n_components  # If no feature selection, return all PCA components\n",
        "\n",
        "# Initialize best_tpot and best_recall\n",
        "best_tpot = None\n",
        "best_recall = 0\n",
        "\n",
        "# Set the number of CPU cores to use\n",
        "n_jobs = 4\n",
        "\n",
        "# Set the sample size for TPOT\n",
        "tpot_sample_size = 5000  # Adjust this value based on your available memory\n",
        "\n",
        "# Set the number of components for PCA\n",
        "n_components = 100  # Adjust this value based on your needs\n",
        "\n",
        "for run in range(n_runs):\n",
        "    print(f\"\\nStarting run {run + 1}/{n_runs}\")\n",
        "    start_time = time.time()\n",
        "\n",
        "    # Split data for this run\n",
        "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42 + run)\n",
        "\n",
        "    # Sample data for TPOT\n",
        "    if len(X_train) > tpot_sample_size:\n",
        "        X_train_sample, _, y_train_sample, _ = train_test_split(\n",
        "            X_train, y_train, train_size=tpot_sample_size, stratify=y_train, random_state=42 + run\n",
        "        )\n",
        "    else:\n",
        "        X_train_sample, y_train_sample = X_train, y_train\n",
        "\n",
        "    # Create preprocessing pipeline\n",
        "    preprocessor = Pipeline([\n",
        "        ('quantile', QuantileTransformer(n_quantiles=1000, output_distribution='normal')),\n",
        "        ('pca', PCA(n_components=n_components))\n",
        "    ])\n",
        "\n",
        "    # Fit preprocessor on training data and transform both training and test data\n",
        "    X_train_preprocessed = preprocessor.fit_transform(X_train_sample)\n",
        "    X_test_preprocessed = preprocessor.transform(X_test)\n",
        "\n",
        "    # TPOT classifier with memory-optimized settings\n",
        "    tpot = TPOTClassifier(\n",
        "        scoring='recall',\n",
        "        verbosity=2,\n",
        "        generations=5,\n",
        "        population_size=20,\n",
        "        max_time_mins=30,\n",
        "        max_eval_time_mins=5,\n",
        "        random_state=42 + run,\n",
        "        config_dict='TPOT sparse',\n",
        "        memory='auto',\n",
        "        n_jobs=n_jobs,\n",
        "        periodic_checkpoint_folder='tpot_checkpoints',\n",
        "        warm_start=True,\n",
        "        cv=3,\n",
        "        early_stop=3\n",
        "    )\n",
        "\n",
        "    # Fit TPOT on the preprocessed data\n",
        "    tpot.fit(X_train_preprocessed, y_train_sample)\n",
        "\n",
        "    # Clear memory\n",
        "    del X_train_sample, y_train_sample, X_train_preprocessed\n",
        "    gc.collect()\n",
        "\n",
        "    # Predict using the preprocessed test set\n",
        "    y_pred = tpot.predict(X_test_preprocessed)\n",
        "    recall = recall_score(y_test, y_pred, average='macro')\n",
        "\n",
        "    # Update best_tpot if this run has better recall score\n",
        "    if recall > best_recall:\n",
        "        best_recall = recall\n",
        "        best_tpot = tpot\n",
        "\n",
        "    # Get pipeline string and extract number of features\n",
        "    pipeline_str = str(tpot.fitted_pipeline_)\n",
        "    n_features = get_n_features(pipeline_str)\n",
        "\n",
        "    # Store results\n",
        "    results['run'].append(run + 1)\n",
        "    results['recall_score'].append(recall)\n",
        "    results['best_pipeline'].append(pipeline_str)\n",
        "    results['n_features'].append(n_features)\n",
        "    results['runtime'].append(time.time() - start_time)\n",
        "    results['pipelines_tested'].append(len(tpot.evaluated_individuals_))\n",
        "\n",
        "    print(f\"Run {run + 1} completed. Recall Score: {recall:.4f}, PCA Components: {n_components}, Features selected: {n_features}, Pipelines tested: {len(tpot.evaluated_individuals_)}\")\n",
        "\n",
        "    # Clear memory\n",
        "    del tpot, X_test_preprocessed\n",
        "    gc.collect()\n",
        "\n",
        "# Convert results to DataFrame\n",
        "results_df = pd.DataFrame(results)\n",
        "\n",
        "# Print summary statistics\n",
        "print(\"\\nSummary Statistics:\")\n",
        "print(results_df.describe())\n",
        "\n",
        "# Save results to CSV\n",
        "results_df.to_csv('tpot_recall_results.csv', index=False)\n",
        "\n",
        "print(\"\\nResults saved to 'tpot_recall_results.csv'\")\n",
        "print(f\"TPOT sample size used: {tpot_sample_size}\")\n",
        "print(f\"Number of PCA components: {n_components}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 548
        },
        "id": "usxaQgkSbEND",
        "outputId": "77407ba5-e4c0-4f0e-afe1-34916ea4e266"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ImportError",
          "evalue": "cannot import name '_average' from 'sklearn.utils._array_api' (/usr/local/lib/python3.10/dist-packages/sklearn/utils/_array_api.py)",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-d1c7f3b7ccc9>\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQuantileTransformer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecomposition\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPCA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtpot\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTPOTClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrecall_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/decomposition/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextmath\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrandomized_svd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m from ._dict_learning import (\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mDictionaryLearning\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mMiniBatchDictionaryLearning\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/decomposition/_dict_learning.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0m_fit_context\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m )\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear_model\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLasso\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLassoLars\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morthogonal_mp_gram\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_random_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgen_batches\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgen_even_slices\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_param_validation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mHidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mInterval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mStrOptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# complete documentation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_base\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLinearRegression\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_bayes\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mARDRegression\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBayesianRidge\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m from ._coordinate_descent import (\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_base.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     34\u001b[0m )\n\u001b[1;32m     35\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_random_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m from ..utils._array_api import (\n\u001b[0m\u001b[1;32m     37\u001b[0m     \u001b[0m_asarray_with_order\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0m_average\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mImportError\u001b[0m: cannot import name '_average' from 'sklearn.utils._array_api' (/usr/local/lib/python3.10/dist-packages/sklearn/utils/_array_api.py)",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# TPOT Algo Stats: Supervised Learning"
      ],
      "metadata": {
        "id": "KYJ1AkAStACV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert results to DataFrame\n",
        "results_df = pd.DataFrame(results)\n",
        "\n",
        "# Print summary statistics\n",
        "print(\"\\nSummary Statistics:\")\n",
        "print(results_df.describe())\n",
        "\n",
        "# Save results to CSV\n",
        "results_df.to_csv('tpot_results_recall_features_pipelines.csv', index=False)\n",
        "\n",
        "print(\"\\nResults saved to 'tpot_results_recall_features_pipelines.csv'\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZAFOj5Ya7GhW",
        "outputId": "cc6590f9-1398-4aab-8813-2ad645fab113"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Summary Statistics:\n",
            "       run  recall_score  n_features     runtime  pipelines_tested\n",
            "count  3.0      3.000000         3.0    3.000000          3.000000\n",
            "mean   2.0      0.506667       100.0  451.956367         94.666667\n",
            "std    1.0      0.011547         0.0   49.738718          0.577350\n",
            "min    1.0      0.500000       100.0  396.558102         94.000000\n",
            "25%    1.5      0.500000       100.0  431.545281         94.500000\n",
            "50%    2.0      0.500000       100.0  466.532460         95.000000\n",
            "75%    2.5      0.510000       100.0  479.655500         95.000000\n",
            "max    3.0      0.520000       100.0  492.778541         95.000000\n",
            "\n",
            "Results saved to 'tpot_results_recall_features_pipelines.csv'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model export: TPOT's best pipeline, Supervised"
      ],
      "metadata": {
        "id": "vM6TVwHYtKNa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import dill\n",
        "\n",
        "# Assuming `best_tpot` is your optimized TPOT object\n",
        "if best_tpot is not None:\n",
        "    with open('best_tpot_model.pkl', 'wb') as f:\n",
        "        dill.dump(best_tpot.fitted_pipeline_, f)\n",
        "    print(\"TPOT model saved using dill.\")\n",
        "else:\n",
        "    print(\"No valid TPOT model was found.\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ojygkYhE3B_j",
        "outputId": "35da40ee-f656-4b9f-818b-27e7633570d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TPOT model saved using dill.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluation of the model"
      ],
      "metadata": {
        "id": "lqXIlzZ0u7oN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(10, 18))\n",
        "\n",
        "ax1.plot(results_df['run'], results_df['recall_score'], 'bo-')\n",
        "ax1.set_title('TPOT Performance (Recall Score) Across Runs')\n",
        "ax1.set_xlabel('Run')\n",
        "ax1.set_ylabel('Recall Score')\n",
        "ax1.grid(True)\n",
        "\n",
        "ax2.plot(results_df['run'], results_df['n_features'], 'ro-')\n",
        "ax2.set_title('Number of Features Selected Across Runs')\n",
        "ax2.set_xlabel('Run')\n",
        "ax2.set_ylabel('Number of Features')\n",
        "ax2.grid(True)\n",
        "\n",
        "# Changed this line\n",
        "ax3.plot(results_df['run'], results_df['pipelines_tested'], 'go-')\n",
        "ax3.set_title('Number of Pipelines Tested Across Runs')\n",
        "ax3.set_xlabel('Run')\n",
        "ax3.set_ylabel('Number of Pipelines')\n",
        "ax3.grid(True)\n",
        "\n",
        "plt.tight_layout()\n",
        "plt.savefig('tpot_performance_features_pipelines.png')\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "PbOIGDR9cAgM",
        "outputId": "ce825cea-2038-4420-f857-50db6d358292"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x1800 with 3 Axes>"
            ],
            "image/png": "[encoded data, 126432 characters]\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Handling Imbalanced Data"
      ],
      "metadata": {
        "id": "RG5bRw2lt0GD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from imblearn.pipeline import Pipeline\n",
        "from sklearn.metrics import balanced_accuracy_score, roc_auc_score, average_precision_score\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.feature_selection import RFECV\n",
        "from sklearn.preprocessing import QuantileTransformer\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.pipeline import Pipeline as ImbPipeline\n",
        "\n",
        "\n",
        "# Encode labels\n",
        "le = LabelEncoder()\n",
        "df['label_encoded'] = le.fit_transform(df['label'])\n",
        "\n",
        "# Split data\n",
        "X = np.array(df['message_vector'].tolist())\n",
        "y = df['label_encoded'].values\n",
        "\n",
        "# Split data for this run\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "\n",
        "# Feature selection\n",
        "feature_selector = RFECV(\n",
        "    estimator=RandomForestClassifier(n_estimators=10, class_weight='balanced', random_state=42),\n",
        "    step=5000,\n",
        "    min_features_to_select=1000,\n",
        "    cv=3,\n",
        "    scoring='balanced_accuracy',\n",
        "    n_jobs=-1\n",
        ")\n",
        "\n",
        "\n",
        "# Quantile transformer\n",
        "quantile_transformer = QuantileTransformer(n_quantiles=1000, output_distribution='normal', random_state=42)\n",
        "\n",
        "# Dimensionality reduction\n",
        "pca = PCA(n_components=0.95)\n",
        "\n",
        "# Create the pipeline\n",
        "pipeline = ImbPipeline([\n",
        "    ('feature_selection', feature_selector),\n",
        "    ('quantile_transform', quantile_transformer),\n",
        "    ('pca', pca),\n",
        "    ('smote', SMOTE(random_state=42)),\n",
        "    ('classifier', RandomForestClassifier(class_weight='balanced', random_state=42))\n",
        "])\n",
        "\n",
        "# Fit the pipeline and make predictions\n",
        "pipeline.fit(X_train, y_train)\n",
        "y_pred = pipeline.predict(X_test)\n",
        "\n",
        "# Calculate metrics\n",
        "balanced_acc = balanced_accuracy_score(y_test, y_pred)\n",
        "roc_auc = roc_auc_score(y_test, pipeline.predict_proba(X_test)[:, 1])\n",
        "avg_precision = average_precision_score(y_test, pipeline.predict_proba(X_test)[:, 1])\n",
        "\n",
        "print(f\"Balanced Accuracy: {balanced_acc:.4f}\")\n",
        "print(f\"ROC AUC: {roc_auc:.4f}\")\n",
        "print(f\"Average Precision: {avg_precision:.4f}\")\n",
        "\n",
        "# Cross-validation\n",
        "cv_scores = cross_val_score(pipeline, X, y, cv=5, scoring='balanced_accuracy')\n",
        "print(f\"Cross-validation Balanced Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std() * 2:.4f})\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zh-ULZZ0099E",
        "outputId": "9bb9313a-7e4c-40bd-8c64-f78d1c57636d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Balanced Accuracy: 0.6000\n",
            "ROC AUC: 0.9933\n",
            "Average Precision: 0.9999\n",
            "Cross-validation Balanced Accuracy: 0.7230 (+/- 0.1013)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Fit the feature selector and print the number of selected features\n",
        "feature_selector.fit(X_train, y_train)\n",
        "print(f\"Number of features selected: {feature_selector.n_features_}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "brczGHV083MW",
        "outputId": "54081516-0074-47d2-da92-ad921d8500f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of features selected: 25000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Testing the predictions"
      ],
      "metadata": {
        "id": "zCJ3jHyZQIS_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_message_benign = random_good_message\n",
        "test_message_bad = random_bad_message\n",
        "\n",
        "\n",
        "from tokenizers import Tokenizer\n",
        "import torch\n",
        "import numpy as np\n",
        "import polars as pl\n",
        "import numpy as np\n",
        "\n",
        "# Load the custom tokenizer\n",
        "tokenizer = Tokenizer.from_file(\"log_tokenizer.json\")\n",
        "\n",
        "# Define the device (assuming you're using PyTorch and want to specify CPU or GPU)\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "vector_good = vectorize_text(test_message_benign).flatten()\n",
        "print(len(vector_good))\n",
        "\n",
        "\n",
        "# Assuming 'vector' is your input\n",
        "vector_good = np.array(vector_good)\n",
        "vector_2d_good = vector_good.reshape(1, -1)\n",
        "\n",
        "# Class prediction\n",
        "prediction = pipeline.predict(vector_2d_good)\n",
        "print(\"Predicted class:\", prediction[0])\n",
        "\n",
        "# Probability prediction\n",
        "probabilities = pipeline.predict_proba(vector_2d_good)\n",
        "print(\"Class probabilities:\", probabilities[0])\n",
        "\n",
        "\n",
        "vector_bad = vectorize_text(test_message_bad).flatten()\n",
        "print(len(vector_bad))\n",
        "\n",
        "# Assuming 'vector' is your input\n",
        "vector_bad = np.array(vector_bad)\n",
        "vector_2d_bad = vector_bad.reshape(1, -1)\n",
        "\n",
        "# Class prediction\n",
        "prediction = pipeline.predict(vector_2d_bad)\n",
        "print(\"Predicted class:\", prediction[0])\n",
        "\n",
        "# Probability prediction\n",
        "probabilities = pipeline.predict_proba(vector_2d_bad)\n",
        "print(\"Class probabilities:\", probabilities[0])\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3HpxPWlIPHT4",
        "outputId": "4501c78a-74d8-49b7-bc3e-0c647ba5e71f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "30000\n",
            "Predicted class: 1\n",
            "Class probabilities: [0. 1.]\n",
            "30000\n",
            "Predicted class: 0\n",
            "Class probabilities: [0.56 0.44]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Save the Supervised Learning Model (joblib)\n",
        "\n",
        "Generating the model can take roughly 1h of computation time on a modern 8 core / 64 GB RAM server."
      ],
      "metadata": {
        "id": "YVPPGLSA6uZ-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "\n",
        "# Save the model using joblib\n",
        "model_filename = 'manual_model_1.joblib'\n",
        "joblib.dump(pipeline, model_filename)\n",
        "\n",
        "print(f\"\\nModel exported as joblib: {model_filename}\")\n",
        "\n",
        "\n",
        "# Verify the saved model\n",
        "try:\n",
        "    loaded_model = joblib.load(model_filename)\n",
        "except Exception as e:\n",
        "    print(f\"\\nError verifying saved model: {str(e)}\")\n",
        "\n",
        "# Print information about how to use the saved model\n",
        "print(\"\\nTo use the saved model in the future:\")\n",
        "print(\"1. Load the model: loaded_model = joblib.load('best_tpot_nn_model.joblib')\")\n",
        "print(\"2. Make predictions: predictions = loaded_model.predict(X)\")\n",
        "print(\"Note: Make sure to preprocess your input data (X) the same way as during training.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lpbrnU_06Gfi",
        "outputId": "007bae7d-2533-477b-b23f-8699f09321bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Model exported as joblib: manual_model_123.joblib\n",
            "\n",
            "To use the saved model in the future:\n",
            "1. Load the model: loaded_model = joblib.load('best_tpot_nn_model.joblib')\n",
            "2. Make predictions: predictions = loaded_model.predict(X)\n",
            "Note: Make sure to preprocess your input data (X) the same way as during training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Confusion matrix"
      ],
      "metadata": {
        "id": "l1u70hEaAYgD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Make predictions\n",
        "y_pred = pipeline.predict(X_test)\n",
        "\n",
        "# Create the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "\n",
        "# Create a ConfusionMatrixDisplay object\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['good', 'bad'])\n",
        "\n",
        "# Plot the confusion matrix\n",
        "plt.figure(figsize=(10, 7))\n",
        "disp.plot(cmap='Blues', values_format='d')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.show()\n",
        "\n",
        "# Print the confusion matrix\n",
        "print(\"Confusion Matrix:\")\n",
        "print(cm)\n",
        "\n",
        "# Calculate and print additional metrics\n",
        "tn, fp, fn, tp = cm.ravel()\n",
        "precision = tp / (tp + fp)\n",
        "recall = tp / (tp + fn)\n",
        "f1_score = 2 * (precision * recall) / (precision + recall)\n",
        "\n",
        "print(f\"\\nPrecision: {precision:.4f}\")\n",
        "print(f\"Recall: {recall:.4f}\")\n",
        "print(f\"F1-score: {f1_score:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 617
        },
        "id": "5GaLAsg661Qh",
        "outputId": "a546f9f0-9c8f-433c-d5f5-8440a058f840"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "[encoded data, 25844 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Confusion Matrix:\n",
            "[[   5   20]\n",
            " [   0 2666]]\n",
            "\n",
            "Precision: 0.9926\n",
            "Recall: 1.0000\n",
            "F1-score: 0.9963\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# CV scores"
      ],
      "metadata": {
        "id": "0jtLBuvnAeND"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import balanced_accuracy_score, roc_auc_score, average_precision_score\n",
        "import numpy as np\n",
        "\n",
        "# Define the number of folds\n",
        "n_splits = 5\n",
        "\n",
        "# Create a StratifiedKFold object\n",
        "skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
        "\n",
        "# Initialize lists to store the scores\n",
        "balanced_accuracy_scores = []\n",
        "roc_auc_scores = []\n",
        "average_precision_scores = []\n",
        "\n",
        "# Perform cross-validation manuall\n",
        "for fold, (train_index, val_index) in enumerate(skf.split(X, y), 1):\n",
        "    X_train, X_val = X[train_index], X[val_index]\n",
        "    y_train, y_val = y[train_index], y[val_index]\n",
        "\n",
        "    # Fit the pipeline\n",
        "    pipeline.fit(X_train, y_train)\n",
        "\n",
        "    # Make predictions\n",
        "    y_pred = pipeline.predict(X_val)\n",
        "    y_pred_proba = pipeline.predict_proba(X_val)[:, 1]\n",
        "\n",
        "    # Calculate scores\n",
        "    balanced_accuracy_scores.append(balanced_accuracy_score(y_val, y_pred))\n",
        "    roc_auc_scores.append(roc_auc_score(y_val, y_pred_proba))\n",
        "    average_precision_scores.append(average_precision_score(y_val, y_pred_proba))\n",
        "\n",
        "    print(f\"Fold {fold}:\")\n",
        "    print(f\"  Balanced Accuracy: {balanced_accuracy_scores[-1]:.4f}\")\n",
        "    print(f\"  ROC AUC: {roc_auc_scores[-1]:.4f}\")\n",
        "    print(f\"  Average Precision: {average_precision_scores[-1]:.4f}\")\n",
        "\n",
        "# Print the overall results\n",
        "print(\"\\nCross-validation results:\")\n",
        "print(f\"Balanced Accuracy: {np.mean(balanced_accuracy_scores):.4f} (+/- {np.std(balanced_accuracy_scores) * 2:.4f})\")\n",
        "print(f\"ROC AUC: {np.mean(roc_auc_scores):.4f} (+/- {np.std(roc_auc_scores) * 2:.4f})\")\n",
        "print(f\"Average Precision: {np.mean(average_precision_scores):.4f} (+/- {np.std(average_precision_scores) * 2:.4f})\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cZqWaRDr61mL",
        "outputId": "d77b060e-d380-4978-c920-97d260f2cf64"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fold 1:\n",
            "  Balanced Accuracy: 0.7493\n",
            "  ROC AUC: 0.9968\n",
            "  Average Precision: 1.0000\n",
            "Fold 2:\n",
            "  Balanced Accuracy: 0.7605\n",
            "  ROC AUC: 0.9944\n",
            "  Average Precision: 0.9999\n",
            "Fold 3:\n",
            "  Balanced Accuracy: 0.7172\n",
            "  ROC AUC: 0.9915\n",
            "  Average Precision: 0.9999\n",
            "Fold 4:\n",
            "  Balanced Accuracy: 0.7170\n",
            "  ROC AUC: 0.9932\n",
            "  Average Precision: 0.9999\n",
            "Fold 5:\n",
            "  Balanced Accuracy: 0.6953\n",
            "  ROC AUC: 0.9957\n",
            "  Average Precision: 1.0000\n",
            "\n",
            "Cross-validation results:\n",
            "Balanced Accuracy: 0.7278 (+/- 0.0475)\n",
            "ROC AUC: 0.9943 (+/- 0.0037)\n",
            "Average Precision: 0.9999 (+/- 0.0000)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Grid search optimization"
      ],
      "metadata": {
        "id": "_XUIxpRdKzf9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import cross_val_score\n",
        "import itertools\n",
        "\n",
        "# Define parameter grid\n",
        "param_grid = {\n",
        "    'n_estimators': [100, 200, 300],\n",
        "    'max_depth': [None, 10, 20, 30],\n",
        "    'min_samples_split': [2, 5, 10],\n",
        "    'min_samples_leaf': [1, 2, 4]\n",
        "}\n",
        "\n",
        "# Generate all combinations of parameters\n",
        "param_combinations = list(itertools.product(*param_grid.values()))\n",
        "\n",
        "best_score = -float('inf')\n",
        "best_params = None\n",
        "\n",
        "# Manually perform grid search\n",
        "for params in param_combinations:\n",
        "    # Create a new pipeline with current parameters\n",
        "    current_pipeline = Pipeline([\n",
        "        ('smote', SMOTE(sampling_strategy=0.35, random_state=42)),\n",
        "        ('undersampler', RandomUnderSampler(sampling_strategy=1.0, random_state=42)),\n",
        "        ('classifier', RandomForestClassifier(\n",
        "            n_estimators=params[0],\n",
        "            max_depth=params[1],\n",
        "            min_samples_split=params[2],\n",
        "            min_samples_leaf=params[3],\n",
        "            class_weight='balanced',\n",
        "            random_state=42\n",
        "        ))\n",
        "    ])\n",
        "\n",
        "    # Perform cross-validation\n",
        "    scores = cross_val_score(current_pipeline, X, y, cv=5, scoring='balanced_accuracy')\n",
        "    mean_score = scores.mean()\n",
        "\n",
        "    # Update best score and parameters if current is better\n",
        "    if mean_score > best_score:\n",
        "        best_score = mean_score\n",
        "        best_params = params\n",
        "\n",
        "    print(f\"Parameters: {params}, Score: {mean_score}\")\n",
        "\n",
        "print(f\"\\nBest parameters: {best_params}\")\n",
        "print(f\"Best cross-validation score: {best_score}\")\n",
        "\n",
        "# Create the best pipeline\n",
        "best_pipeline = Pipeline([\n",
        "    ('smote', SMOTE(sampling_strategy=0.35, random_state=42)),\n",
        "    ('undersampler', RandomUnderSampler(sampling_strategy=1.0, random_state=42)),\n",
        "    ('classifier', RandomForestClassifier(\n",
        "        n_estimators=best_params[0],\n",
        "        max_depth=best_params[1],\n",
        "        min_samples_split=best_params[2],\n",
        "        min_samples_leaf=best_params[3],\n",
        "        class_weight='balanced',\n",
        "        random_state=42\n",
        "    ))\n",
        "])\n",
        "\n",
        "# Fit the best pipeline\n",
        "best_pipeline.fit(X, y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "yIeeLfSlBXI7",
        "outputId": "ee84da33-1d68-4cc3-8f82-82c20b4c2846"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Parameters: (100, None, 2, 1), Score: 0.6601958996148235\n",
            "Parameters: (100, None, 2, 2), Score: 0.6575723238043774\n",
            "Parameters: (100, None, 2, 4), Score: 0.6832200924981155\n",
            "Parameters: (100, None, 5, 1), Score: 0.6550882823515521\n",
            "Parameters: (100, None, 5, 2), Score: 0.6617703512425128\n",
            "Parameters: (100, None, 5, 4), Score: 0.6832200924981155\n",
            "Parameters: (100, None, 10, 1), Score: 0.6531768364259575\n",
            "Parameters: (100, None, 10, 2), Score: 0.6652185568886595\n",
            "Parameters: (100, None, 10, 4), Score: 0.6606836474980943\n",
            "Parameters: (100, 10, 2, 1), Score: 0.7538128573575538\n",
            "Parameters: (100, 10, 2, 2), Score: 0.7490530042380288\n",
            "Parameters: (100, 10, 2, 4), Score: 0.7528384007866634\n",
            "Parameters: (100, 10, 5, 1), Score: 0.7531383491639173\n",
            "Parameters: (100, 10, 5, 2), Score: 0.7575612641582747\n",
            "Parameters: (100, 10, 5, 4), Score: 0.7528384007866634\n",
            "Parameters: (100, 10, 10, 1), Score: 0.7454542837406529\n",
            "Parameters: (100, 10, 10, 2), Score: 0.7450421584057931\n",
            "Parameters: (100, 10, 10, 4), Score: 0.7454544241724976\n",
            "Parameters: (100, 20, 2, 1), Score: 0.667890060811074\n",
            "Parameters: (100, 20, 2, 2), Score: 0.6709738726297466\n",
            "Parameters: (100, 20, 2, 4), Score: 0.6855542931725662\n",
            "Parameters: (100, 20, 5, 1), Score: 0.6718902772293134\n",
            "Parameters: (100, 20, 5, 2), Score: 0.6761462302501122\n",
            "Parameters: (100, 20, 5, 4), Score: 0.6855542931725662\n",
            "Parameters: (100, 20, 10, 1), Score: 0.6663908946953685\n",
            "Parameters: (100, 20, 10, 2), Score: 0.6653415739080288\n",
            "Parameters: (100, 20, 10, 4), Score: 0.6765587768805064\n",
            "Parameters: (100, 30, 2, 1), Score: 0.659548594346173\n",
            "Parameters: (100, 30, 2, 2), Score: 0.6622950046145905\n",
            "Parameters: (100, 30, 2, 4), Score: 0.6784974819038248\n",
            "Parameters: (100, 30, 5, 1), Score: 0.6547135399737714\n",
            "Parameters: (100, 30, 5, 2), Score: 0.6574599783285577\n",
            "Parameters: (100, 30, 5, 4), Score: 0.6784974819038248\n",
            "Parameters: (100, 30, 10, 1), Score: 0.6532142895989589\n",
            "Parameters: (100, 30, 10, 2), Score: 0.6739245014401413\n",
            "Parameters: (100, 30, 10, 4), Score: 0.665106450146976\n",
            "Parameters: (200, None, 2, 1), Score: 0.6643938427938523\n",
            "Parameters: (200, None, 2, 2), Score: 0.6576473705822248\n",
            "Parameters: (200, None, 2, 4), Score: 0.6868933120891442\n",
            "Parameters: (200, None, 5, 1), Score: 0.6553131418214051\n",
            "Parameters: (200, None, 5, 2), Score: 0.6623325701330676\n",
            "Parameters: (200, None, 5, 4), Score: 0.6868933120891442\n",
            "Parameters: (200, None, 10, 1), Score: 0.6578994329770638\n",
            "Parameters: (200, None, 10, 2), Score: 0.6652935615369535\n",
            "Parameters: (200, None, 10, 4), Score: 0.6698392267301753\n",
            "Parameters: (200, 10, 2, 1), Score: 0.745267074048384\n",
            "Parameters: (200, 10, 2, 2), Score: 0.7449298971890804\n",
            "Parameters: (200, 10, 2, 4), Score: 0.7447048832441983\n",
            "Parameters: (200, 10, 5, 1), Score: 0.7492776671032991\n",
            "Parameters: (200, 10, 5, 2), Score: 0.74897785915789\n",
            "Parameters: (200, 10, 5, 4), Score: 0.7447048832441983\n",
            "Parameters: (200, 10, 10, 1), Score: 0.7457916712477236\n",
            "Parameters: (200, 10, 10, 2), Score: 0.7454544241724976\n",
            "Parameters: (200, 10, 10, 4), Score: 0.7451547004861954\n",
            "Parameters: (200, 20, 2, 1), Score: 0.6638045192806029\n",
            "Parameters: (200, 20, 2, 2), Score: 0.6673279261796262\n",
            "Parameters: (200, 20, 2, 4), Score: 0.6721734274046259\n",
            "Parameters: (200, 20, 5, 1), Score: 0.6592693009399793\n",
            "Parameters: (200, 20, 5, 2), Score: 0.6672631436929786\n",
            "Parameters: (200, 20, 5, 4), Score: 0.6721734274046259\n",
            "Parameters: (200, 20, 10, 1), Score: 0.6754715254518933\n",
            "Parameters: (200, 20, 10, 2), Score: 0.6794446232042534\n",
            "Parameters: (200, 20, 10, 4), Score: 0.6768585286531775\n",
            "Parameters: (200, 30, 2, 1), Score: 0.6599233788535073\n",
            "Parameters: (200, 30, 2, 2), Score: 0.6574974595879279\n",
            "Parameters: (200, 30, 2, 4), Score: 0.6783100896501577\n",
            "Parameters: (200, 30, 5, 1), Score: 0.6550133198328116\n",
            "Parameters: (200, 30, 5, 2), Score: 0.6624449717816251\n",
            "Parameters: (200, 30, 5, 4), Score: 0.6783100896501577\n",
            "Parameters: (200, 30, 10, 1), Score: 0.6581617737062869\n",
            "Parameters: (200, 30, 10, 2), Score: 0.6695289298026146\n",
            "Parameters: (200, 30, 10, 4), Score: 0.6699142032921004\n",
            "Parameters: (300, None, 2, 1), Score: 0.6602333949173783\n",
            "Parameters: (300, None, 2, 2), Score: 0.6577598143603358\n",
            "Parameters: (300, None, 2, 4), Score: 0.6747867810534257\n",
            "Parameters: (300, None, 5, 1), Score: 0.6598858835509525\n",
            "Parameters: (300, None, 5, 2), Score: 0.662444971781625\n",
            "Parameters: (300, None, 5, 4), Score: 0.6747867810534257\n",
            "Parameters: (300, None, 10, 1), Score: 0.6583491800031385\n",
            "Parameters: (300, None, 10, 2), Score: 0.6609831745798139\n",
            "Parameters: (300, None, 10, 4), Score: 0.6698495331509285\n",
            "Parameters: (300, 10, 2, 1), Score: 0.7449297427140511\n",
            "Parameters: (300, 10, 2, 2), Score: 0.7497272877407137\n",
            "Parameters: (300, 10, 2, 4), Score: 0.7446298083799818\n",
            "Parameters: (300, 10, 5, 1), Score: 0.7494274517089357\n",
            "Parameters: (300, 10, 5, 2), Score: 0.7490527514607083\n",
            "Parameters: (300, 10, 5, 4), Score: 0.7446298083799818\n",
            "Parameters: (300, 10, 10, 1), Score: 0.7452669336165393\n",
            "Parameters: (300, 10, 10, 2), Score: 0.7453419242216488\n",
            "Parameters: (300, 10, 10, 4), Score: 0.7447797474606476\n",
            "Parameters: (300, 20, 2, 1), Score: 0.6591841443459611\n",
            "Parameters: (300, 20, 2, 2), Score: 0.6669530854995541\n",
            "Parameters: (300, 20, 2, 4), Score: 0.6768960941716546\n",
            "Parameters: (300, 20, 5, 1), Score: 0.6676651451684832\n",
            "Parameters: (300, 20, 5, 2), Score: 0.6758463520887806\n",
            "Parameters: (300, 20, 5, 4), Score: 0.6768960941716546\n",
            "Parameters: (300, 20, 10, 1), Score: 0.6711509725058449\n",
            "Parameters: (300, 20, 10, 2), Score: 0.6749843392960013\n",
            "Parameters: (300, 20, 10, 4), Score: 0.6810939530915764\n",
            "Parameters: (300, 30, 2, 1), Score: 0.6600357805020648\n",
            "Parameters: (300, 30, 2, 2), Score: 0.6577223471441499\n",
            "Parameters: (300, 30, 2, 4), Score: 0.6746743653616838\n",
            "Parameters: (300, 30, 5, 1), Score: 0.6596235287785447\n",
            "Parameters: (300, 30, 5, 2), Score: 0.662145135749847\n",
            "Parameters: (300, 30, 5, 4), Score: 0.6746743653616838\n",
            "Parameters: (300, 30, 10, 1), Score: 0.6583491800031385\n",
            "Parameters: (300, 30, 10, 2), Score: 0.660870730801703\n",
            "Parameters: (300, 30, 10, 4), Score: 0.6699244816264847\n",
            "\n",
            "Best parameters: (100, 10, 5, 2)\n",
            "Best cross-validation score: 0.7575612641582747\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(steps=[('smote', SMOTE(random_state=42, sampling_strategy=0.35)),\n",
              "                ('undersampler',\n",
              "                 RandomUnderSampler(random_state=42, sampling_strategy=1.0)),\n",
              "                ('classifier',\n",
              "                 RandomForestClassifier(class_weight='balanced', max_depth=10,\n",
              "                                        min_samples_leaf=2, min_samples_split=5,\n",
              "                                        random_state=42))])"
            ],
            "text/html": [
              "<style>#sk-container-id-10 {color: black;}#sk-container-id-10 pre{padding: 0;}#sk-container-id-10 div.sk-toggleable {background-color: white;}#sk-container-id-10 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-10 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-10 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-10 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-10 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-10 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-10 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-10 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-10 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-10 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-10 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-10 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-10 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-10 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-10 div.sk-item {position: relative;z-index: 1;}#sk-container-id-10 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-10 div.sk-item::before, #sk-container-id-10 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-10 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-10 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-10 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-10 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-10 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-10 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-10 div.sk-label-container {text-align: center;}#sk-container-id-10 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-10 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-10\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;smote&#x27;, SMOTE(random_state=42, sampling_strategy=0.35)),\n",
              "                (&#x27;undersampler&#x27;,\n",
              "                 RandomUnderSampler(random_state=42, sampling_strategy=1.0)),\n",
              "                (&#x27;classifier&#x27;,\n",
              "                 RandomForestClassifier(class_weight=&#x27;balanced&#x27;, max_depth=10,\n",
              "                                        min_samples_leaf=2, min_samples_split=5,\n",
              "                                        random_state=42))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-37\" type=\"checkbox\" ><label for=\"sk-estimator-id-37\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;smote&#x27;, SMOTE(random_state=42, sampling_strategy=0.35)),\n",
              "                (&#x27;undersampler&#x27;,\n",
              "                 RandomUnderSampler(random_state=42, sampling_strategy=1.0)),\n",
              "                (&#x27;classifier&#x27;,\n",
              "                 RandomForestClassifier(class_weight=&#x27;balanced&#x27;, max_depth=10,\n",
              "                                        min_samples_leaf=2, min_samples_split=5,\n",
              "                                        random_state=42))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-38\" type=\"checkbox\" ><label for=\"sk-estimator-id-38\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SMOTE</label><div class=\"sk-toggleable__content\"><pre>SMOTE(random_state=42, sampling_strategy=0.35)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-39\" type=\"checkbox\" ><label for=\"sk-estimator-id-39\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomUnderSampler</label><div class=\"sk-toggleable__content\"><pre>RandomUnderSampler(random_state=42, sampling_strategy=1.0)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-40\" type=\"checkbox\" ><label for=\"sk-estimator-id-40\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(class_weight=&#x27;balanced&#x27;, max_depth=10,\n",
              "                       min_samples_leaf=2, min_samples_split=5,\n",
              "                       random_state=42)</pre></div></div></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "Pipeline(steps=[('smote', SMOTE(random_state=42, sampling_strategy=0.35)),\n",
        "                ('undersampler',\n",
        "                 RandomUnderSampler(random_state=42, sampling_strategy=1.0)),\n",
        "                ('classifier',\n",
        "                 RandomForestClassifier(class_weight='balanced', max_depth=10,\n",
        "                                        min_samples_leaf=2, min_samples_split=5,\n",
        "                                        random_state=42))])\n",
        "\n",
        "SMOTE\n",
        "SMOTE(random_state=42, sampling_strategy=0.35)\n",
        "\n",
        "RandomUnderSampler\n",
        "RandomUnderSampler(random_state=42, sampling_strategy=1.0)\n",
        "\n",
        "RandomForestClassifier\n",
        "RandomForestClassifier(class_weight='balanced', max_depth=10,\n",
        "                       min_samples_leaf=2, min_samples_split=5,\n",
        "                       random_state=42)\n",
        "\"\"\"\n",
        "\n",
        "from imblearn.pipeline import make_pipeline\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report, balanced_accuracy_score\n",
        "\n",
        "# Assuming you have your features X and target y already defined\n",
        "# Encode labels\n",
        "le = LabelEncoder()\n",
        "df['label_encoded'] = le.fit_transform(df['label'])\n",
        "\n",
        "# Split data\n",
        "X = np.array(df['message_vector'].tolist())\n",
        "y = df['label_encoded'].values\n",
        "\n",
        "\n",
        "# Split the data into training and testing sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Split the data into training and testing sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Split the data into training and testing sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Create the pipeline using make_pipeline\n",
        "pipeline = make_pipeline(\n",
        "    SMOTE(random_state=42, sampling_strategy=0.35),\n",
        "    RandomUnderSampler(random_state=42, sampling_strategy=1.0),\n",
        "    RandomForestClassifier(\n",
        "        class_weight='balanced',\n",
        "        max_depth=10,\n",
        "        min_samples_leaf=2,\n",
        "        min_samples_split=5,\n",
        "        random_state=42\n",
        "    )\n",
        ")\n",
        "\n",
        "# Fit the pipeline\n",
        "pipeline.fit(X_train, y_train)\n",
        "\n",
        "# Use the fitted pipeline to make predictions\n",
        "y_pred = pipeline.predict(X_test)\n",
        "\n",
        "# Evaluate the model\n",
        "print(classification_report(y_test, y_pred))\n",
        "print(f\"Balanced Accuracy: {balanced_accuracy_score(y_test, y_pred)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saqEW4G2HbI3",
        "outputId": "d149844d-0cf7-4e33-89d9-cadf77890678"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.78      0.72      0.75        25\n",
            "           1       1.00      1.00      1.00      2666\n",
            "\n",
            "    accuracy                           1.00      2691\n",
            "   macro avg       0.89      0.86      0.87      2691\n",
            "weighted avg       1.00      1.00      1.00      2691\n",
            "\n",
            "Balanced Accuracy: 0.8590622655663915\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Make predictions\n",
        "y_pred = pipeline.predict(X_test)\n",
        "\n",
        "# Create the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "\n",
        "# Create a ConfusionMatrixDisplay object\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['good', 'bad'])\n",
        "\n",
        "# Plot the confusion matrix\n",
        "plt.figure(figsize=(10, 7))\n",
        "disp.plot(cmap='Blues', values_format='d')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.show()\n",
        "\n",
        "# Print the confusion matrix\n",
        "print(\"Confusion Matrix:\")\n",
        "print(cm)\n",
        "\n",
        "# Calculate and print additional metrics\n",
        "tn, fp, fn, tp = cm.ravel()\n",
        "precision = tp / (tp + fp)\n",
        "recall = tp / (tp + fn)\n",
        "f1_score = 2 * (precision * recall) / (precision + recall)\n",
        "\n",
        "print(f\"\\nPrecision: {precision:.4f}\")\n",
        "print(f\"Recall: {recall:.4f}\")\n",
        "print(f\"F1-score: {f1_score:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 617
        },
        "id": "raf1OKv4_E23",
        "outputId": "6f7ab778-99af-401d-a8e0-2d8bff230902"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "[encoded data, 25104 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Confusion Matrix:\n",
            "[[  18    7]\n",
            " [   5 2661]]\n",
            "\n",
            "Precision: 0.9974\n",
            "Recall: 0.9981\n",
            "F1-score: 0.9978\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import balanced_accuracy_score, roc_auc_score, average_precision_score\n",
        "import numpy as np\n",
        "\n",
        "# Define the number of folds\n",
        "n_splits = 5\n",
        "\n",
        "# Create a StratifiedKFold object\n",
        "skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
        "\n",
        "# Initialize lists to store the scores\n",
        "balanced_accuracy_scores = []\n",
        "roc_auc_scores = []\n",
        "average_precision_scores = []\n",
        "\n",
        "# Perform cross-validation manually\n",
        "for fold, (train_index, val_index) in enumerate(skf.split(X, y), 1):\n",
        "    X_train, X_val = X[train_index], X[val_index]\n",
        "    y_train, y_val = y[train_index], y[val_index]\n",
        "\n",
        "    # Fit the pipeline\n",
        "    pipeline.fit(X_train, y_train)\n",
        "\n",
        "    # Make predictions\n",
        "    y_pred = pipeline.predict(X_val)\n",
        "    y_pred_proba = pipeline.predict_proba(X_val)[:, 1]\n",
        "\n",
        "    # Calculate scores\n",
        "    balanced_accuracy_scores.append(balanced_accuracy_score(y_val, y_pred))\n",
        "    roc_auc_scores.append(roc_auc_score(y_val, y_pred_proba))\n",
        "    average_precision_scores.append(average_precision_score(y_val, y_pred_proba))\n",
        "\n",
        "    print(f\"Fold {fold}:\")\n",
        "    print(f\"  Balanced Accuracy: {balanced_accuracy_scores[-1]:.4f}\")\n",
        "    print(f\"  ROC AUC: {roc_auc_scores[-1]:.4f}\")\n",
        "    print(f\"  Average Precision: {average_precision_scores[-1]:.4f}\")\n",
        "\n",
        "# Print the overall results\n",
        "print(\"\\nCross-validation results:\")\n",
        "print(f\"Balanced Accuracy: {np.mean(balanced_accuracy_scores):.4f} (+/- {np.std(balanced_accuracy_scores) * 2:.4f})\")\n",
        "print(f\"ROC AUC: {np.mean(roc_auc_scores):.4f} (+/- {np.std(roc_auc_scores) * 2:.4f})\")\n",
        "print(f\"Average Precision: {np.mean(average_precision_scores):.4f} (+/- {np.std(average_precision_scores) * 2:.4f})\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WAJiYO5dK7eL",
        "outputId": "21d68432-3b20-4fca-9775-bdf302ce8c0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fold 1:\n",
            "  Balanced Accuracy: 0.9761\n",
            "  ROC AUC: 0.9994\n",
            "  Average Precision: 1.0000\n",
            "Fold 2:\n",
            "  Balanced Accuracy: 0.8679\n",
            "  ROC AUC: 0.9965\n",
            "  Average Precision: 1.0000\n",
            "Fold 3:\n",
            "  Balanced Accuracy: 0.8686\n",
            "  ROC AUC: 0.9984\n",
            "  Average Precision: 1.0000\n",
            "Fold 4:\n",
            "  Balanced Accuracy: 0.8692\n",
            "  ROC AUC: 0.9970\n",
            "  Average Precision: 1.0000\n",
            "Fold 5:\n",
            "  Balanced Accuracy: 0.9115\n",
            "  ROC AUC: 0.9979\n",
            "  Average Precision: 1.0000\n",
            "\n",
            "Cross-validation results:\n",
            "Balanced Accuracy: 0.8987 (+/- 0.0843)\n",
            "ROC AUC: 0.9979 (+/- 0.0020)\n",
            "Average Precision: 1.0000 (+/- 0.0000)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_message_benign = random_good_message\n",
        "test_message_bad = random_bad_message\n",
        "\n",
        "\n",
        "from tokenizers import Tokenizer\n",
        "import torch\n",
        "import numpy as np\n",
        "import polars as pl\n",
        "import numpy as np\n",
        "\n",
        "# Load the custom tokenizer\n",
        "tokenizer = Tokenizer.from_file(\"log_tokenizer.json\")\n",
        "\n",
        "# Define the device (assuming you're using PyTorch and want to specify CPU or GPU)\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "vector_good = vectorize_text(test_message_benign).flatten()\n",
        "print(len(vector_good))\n",
        "\n",
        "\n",
        "# Assuming 'vector' is your input\n",
        "vector_good = np.array(vector_good)\n",
        "vector_2d_good = vector_good.reshape(1, -1)\n",
        "\n",
        "# Class prediction\n",
        "prediction = pipeline.predict(vector_2d_good)\n",
        "print(\"Predicted class:\", prediction[0])\n",
        "\n",
        "# Probability prediction\n",
        "probabilities = pipeline.predict_proba(vector_2d_good)\n",
        "print(\"Class probabilities:\", probabilities[0])\n",
        "\n",
        "\n",
        "vector_bad = vectorize_text(test_message_bad).flatten()\n",
        "print(len(vector_bad))\n",
        "\n",
        "# Assuming 'vector' is your input\n",
        "vector_bad = np.array(vector_bad)\n",
        "vector_2d_bad = vector_bad.reshape(1, -1)\n",
        "\n",
        "# Class prediction\n",
        "prediction = pipeline.predict(vector_2d_bad)\n",
        "print(\"Predicted class:\", prediction[0])\n",
        "\n",
        "# Probability prediction\n",
        "probabilities = pipeline.predict_proba(vector_2d_bad)\n",
        "print(\"Class probabilities:\", probabilities[0])\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RbXSx4Go-6pd",
        "outputId": "6fbe80e4-4818-42dd-d5b6-895cfbcce41f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "30000\n",
            "Predicted class: 1\n",
            "Class probabilities: [0. 1.]\n",
            "30000\n",
            "Predicted class: 0\n",
            "Class probabilities: [0.68148054 0.31851946]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building a Supervised Learning classifier with Tpot (scikit Algorithms) - or an imbalanced dataset"
      ],
      "metadata": {
        "id": "ZWRH59wKLaAX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import make_scorer, confusion_matrix, classification_report\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from imblearn.pipeline import Pipeline as ImbPipeline\n",
        "from tpot import TPOTClassifier\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "\n",
        "def print_step_info(X, y, step_name):\n",
        "    print(f\"\\n--- {step_name} ---\")\n",
        "    print(f\"X shape: {X.shape}\")\n",
        "    print(f\"y shape: {y.shape}\")\n",
        "    print(f\"Class distribution: {np.bincount(y)}\")\n",
        "\n",
        "def custom_scorer(y_true, y_pred):\n",
        "    cm = confusion_matrix(y_true, y_pred)\n",
        "    tn, fp, fn, tp = cm.ravel()\n",
        "    if (tp + fn) == 0 or (tn + fp) == 0:\n",
        "        return 0.0\n",
        "    recall = tp / (tp + fn + 1e-8)\n",
        "    specificity = tn / (tn + fp + 1e-8)\n",
        "    recall_weight = 1.0\n",
        "    specificity_weight = 2.0\n",
        "    score = (recall_weight * recall + specificity_weight * specificity) / (recall_weight + specificity_weight)\n",
        "    return score\n",
        "\n",
        "custom_scorer_obj = make_scorer(custom_scorer, greater_is_better=True)\n",
        "\n",
        "# Assuming you have your features X and target y already defined\n",
        "# Encode labels\n",
        "le = LabelEncoder()\n",
        "df['label_encoded'] = le.fit_transform(df['label'])\n",
        "\n",
        "# Split data\n",
        "X = np.array(df['message_vector'].tolist())\n",
        "y = df['label_encoded'].values\n",
        "\n",
        "print_step_info(X, y, \"Original Data\")\n",
        "\n",
        "# Step 1: Preprocessing\n",
        "imputer = SimpleImputer(strategy='mean')\n",
        "scaler = StandardScaler()\n",
        "pca = TruncatedSVD(n_components=100, random_state=42)  # Reduce to 100 components\n",
        "\n",
        "X = imputer.fit_transform(X)\n",
        "X = scaler.fit_transform(X)\n",
        "X = pca.fit_transform(X)\n",
        "\n",
        "print_step_info(X, y, \"After Preprocessing\")\n",
        "\n",
        "# Step 2: Split the data\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n",
        "print_step_info(X_train, y_train, \"Training Data\")\n",
        "\n",
        "# Step 3: Initial model on imbalanced data\n",
        "initial_model = RandomForestClassifier(n_estimators=100, class_weight='balanced', random_state=42, n_jobs=-1)\n",
        "initial_model.fit(X_train, y_train)\n",
        "y_pred_initial = initial_model.predict(X_test)\n",
        "print(\"\\nInitial Model Performance:\")\n",
        "print(classification_report(y_test, y_pred_initial, zero_division=0))\n",
        "print(\"Custom Score:\", custom_scorer(y_test, y_pred_initial))\n",
        "\n",
        "# Step 4: Resampling\n",
        "smote = SMOTE(sampling_strategy=0.5, random_state=42)\n",
        "rus = RandomUnderSampler(sampling_strategy=0.5, random_state=42)\n",
        "X_resampled, y_resampled = rus.fit_resample(*smote.fit_resample(X_train, y_train))\n",
        "print_step_info(X_resampled, y_resampled, \"Resampled Data\")\n",
        "\n",
        "# Step 5: Model on resampled data\n",
        "resampled_model = RandomForestClassifier(n_estimators=100, class_weight='balanced', random_state=42, n_jobs=-1)\n",
        "resampled_model.fit(X_resampled, y_resampled)\n",
        "y_pred_resampled = resampled_model.predict(X_test)\n",
        "print(\"\\nResampled Model Performance:\")\n",
        "print(classification_report(y_test, y_pred_resampled, zero_division=0))\n",
        "print(\"Custom Score:\", custom_scorer(y_test, y_pred_resampled))\n",
        "\n",
        "# Step 6: Create a new dataset with original minority samples and top predicted minority samples\n",
        "y_pred_proba = resampled_model.predict_proba(X_train)[:, 1]\n",
        "minority_indices = np.where(y_train == 0)[0]\n",
        "predicted_minority_indices = np.argsort(y_pred_proba)[-len(minority_indices)*10:]\n",
        "selected_indices = np.union1d(minority_indices, predicted_minority_indices)\n",
        "\n",
        "X_selected = X_train[selected_indices]\n",
        "y_selected = y_train[selected_indices]\n",
        "print_step_info(X_selected, y_selected, \"Selected Data for Final Model\")\n",
        "\n",
        "# Step 7: Final TPOT optimization\n",
        "tpot_config = {\n",
        "    'sklearn.ensemble.RandomForestClassifier': {\n",
        "        'n_estimators': [100, 200, 500],\n",
        "        'max_depth': [None, 5, 10, 20],\n",
        "        'min_samples_split': [2, 5, 10],\n",
        "        'min_samples_leaf': [1, 2, 4],\n",
        "        'class_weight': ['balanced', 'balanced_subsample']\n",
        "    },\n",
        "    'xgboost.XGBClassifier': {\n",
        "        'n_estimators': [100, 200, 500],\n",
        "        'max_depth': [3, 5, 7, 9],\n",
        "        'learning_rate': [0.01, 0.1, 0.3],\n",
        "        'subsample': [0.5, 0.7, 1.0],\n",
        "        'scale_pos_weight': [1, 10, 100]\n",
        "    },\n",
        "    'imblearn.ensemble.BalancedRandomForestClassifier': {\n",
        "        'n_estimators': [100, 200, 500],\n",
        "        'replacement': [True, False]\n",
        "    },\n",
        "    'imblearn.ensemble.EasyEnsembleClassifier': {\n",
        "        'n_estimators': [10, 20, 30, 50],\n",
        "        'replacement': [True, False]\n",
        "    }\n",
        "}\n",
        "\n",
        "tpot = TPOTClassifier(\n",
        "    config_dict=tpot_config,\n",
        "    generations=1,\n",
        "    population_size=20,\n",
        "    verbosity=2,\n",
        "    scoring=custom_scorer_obj,\n",
        "    random_state=42,\n",
        "    n_jobs=-1,\n",
        "    max_time_mins=60,\n",
        "    max_eval_time_mins=10\n",
        ")\n",
        "\n",
        "try:\n",
        "    tpot.fit(X_selected, y_selected)\n",
        "    y_pred_tpot = tpot.predict(X_test)\n",
        "    print(\"\\nTPOT Model Performance:\")\n",
        "    print(classification_report(y_test, y_pred_tpot, zero_division=0))\n",
        "    print(\"Custom Score:\", custom_scorer(y_test, y_pred_tpot))\n",
        "    print(\"\\nConfusion Matrix:\")\n",
        "    print(confusion_matrix(y_test, y_pred_tpot))\n",
        "    tpot.export('tpot_best_pipeline.py')\n",
        "    print(\"\\nBest pipeline exported to 'tpot_best_pipeline.py'\")\n",
        "except Exception as e:\n",
        "    print(f\"An error occurred during TPOT optimization: {str(e)}\")\n",
        "\n",
        "# Final evaluation\n",
        "print(\"\\nFinal Evaluation:\")\n",
        "print(\"Initial Model Custom Score:\", custom_scorer(y_test, y_pred_initial))\n",
        "print(\"Resampled Model Custom Score:\", custom_scorer(y_test, y_pred_resampled))\n",
        "if 'y_pred_tpot' in locals():\n",
        "    print(\"TPOT Model Custom Score:\", custom_scorer(y_test, y_pred_tpot))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "0ee9d77e31c343d2b226c9e1c370899a",
            "cefbf3ec854e463eac3047d7f2366a9b",
            "5d5a04539517439c9a47f725bbfca641",
            "487df5716a1f42b9ad08a05300882448",
            "f6f95cf2eb1f43cdbb1be5a1ffc19621",
            "a8b2545d7d94490b8df9d8a63d7fde2d",
            "18b7438e17ef4851aa9c8d0dfc52f2d5",
            "939ebe26aaf74df78c81c255aefcc7e5",
            "c5677148554a4e65b74ae9fc95ae4c37",
            "328be8dce5cf4b2a8418e7693f1829c0",
            "48d9d503aebd44578cb545fdb4b28776"
          ]
        },
        "id": "TnChV008alMz",
        "outputId": "2e7242bb-41eb-4657-9aba-80bf41b9e218"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "--- Original Data ---\n",
            "X shape: (13455, 30000)\n",
            "y shape: (13455,)\n",
            "Class distribution: [  114 13341]\n",
            "\n",
            "--- After Preprocessing ---\n",
            "X shape: (13455, 100)\n",
            "y shape: (13455,)\n",
            "Class distribution: [  114 13341]\n",
            "\n",
            "--- Training Data ---\n",
            "X shape: (10764, 100)\n",
            "y shape: (10764,)\n",
            "Class distribution: [   91 10673]\n",
            "\n",
            "Initial Model Performance:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.00      0.00      0.00        23\n",
            "           1       0.99      1.00      1.00      2668\n",
            "\n",
            "    accuracy                           0.99      2691\n",
            "   macro avg       0.50      0.50      0.50      2691\n",
            "weighted avg       0.98      0.99      0.99      2691\n",
            "\n",
            "Custom Score: 0.333333333332084\n",
            "\n",
            "--- Resampled Data ---\n",
            "X shape: (16008, 100)\n",
            "y shape: (16008,)\n",
            "Class distribution: [ 5336 10672]\n",
            "\n",
            "Resampled Model Performance:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.22      0.36        23\n",
            "           1       0.99      1.00      1.00      2668\n",
            "\n",
            "    accuracy                           0.99      2691\n",
            "   macro avg       1.00      0.61      0.68      2691\n",
            "weighted avg       0.99      0.99      0.99      2691\n",
            "\n",
            "Custom Score: 0.47826086950095603\n",
            "\n",
            "--- Selected Data for Final Model ---\n",
            "X shape: (1001, 100)\n",
            "y shape: (1001,)\n",
            "Class distribution: [ 91 910]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Optimization Progress:   0%|          | 0/20 [00:00<?, ?pipeline/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0ee9d77e31c343d2b226c9e1c370899a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 1 - Current best internal CV score: 0.9886446882605258\n",
            "\n",
            "Best pipeline: BalancedRandomForestClassifier(input_matrix, n_estimators=500, replacement=False)\n",
            "\n",
            "TPOT Model Performance:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.07      1.00      0.14        23\n",
            "           1       1.00      0.89      0.94      2668\n",
            "\n",
            "    accuracy                           0.89      2691\n",
            "   macro avg       0.54      0.95      0.54      2691\n",
            "weighted avg       0.99      0.89      0.94      2691\n",
            "\n",
            "Custom Score: 0.9635182405885926\n",
            "\n",
            "Confusion Matrix:\n",
            "[[  23    0]\n",
            " [ 292 2376]]\n",
            "\n",
            "Best pipeline exported to 'tpot_best_pipeline.py'\n",
            "\n",
            "Final Evaluation:\n",
            "Initial Model Custom Score: 0.333333333332084\n",
            "Resampled Model Custom Score: 0.47826086950095603\n",
            "TPOT Model Custom Score: 0.9635182405885926\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Make predictions\n",
        "y_pred = tpot.predict(X_test)\n",
        "\n",
        "# Create the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "\n",
        "# Create a ConfusionMatrixDisplay object\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['negative', 'positive'])\n",
        "\n",
        "# Plot the confusion matrix\n",
        "plt.figure(figsize=(10, 7))\n",
        "disp.plot(cmap='Blues', values_format='d')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.show()\n",
        "\n",
        "# Print the confusion matrix\n",
        "print(\"Confusion Matrix:\")\n",
        "print(cm)\n",
        "\n",
        "# Calculate and print additional metrics\n",
        "tn, fp, fn, tp = cm.ravel()\n",
        "precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n",
        "recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n",
        "f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
        "\n",
        "print(f\"\\nPrecision: {precision:.4f}\")\n",
        "print(f\"Recall: {recall:.4f}\")\n",
        "print(f\"F1-score: {f1_score:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 617
        },
        "id": "top9wzRCdAKK",
        "outputId": "decb8210-d1fc-4452-bc9c-ca6962caff0f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "[encoded data, 28220 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Confusion Matrix:\n",
            "[[  23    0]\n",
            " [ 292 2376]]\n",
            "\n",
            "Precision: 1.0000\n",
            "Recall: 0.8906\n",
            "F1-score: 0.9421\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.metrics import balanced_accuracy_score, roc_auc_score, average_precision_score\n",
        "import numpy as np\n",
        "\n",
        "# Define the number of folds\n",
        "n_splits = 5\n",
        "\n",
        "# Create a StratifiedKFold object\n",
        "skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
        "\n",
        "# Initialize lists to store the scores\n",
        "balanced_accuracy_scores = []\n",
        "roc_auc_scores = []\n",
        "average_precision_scores = []\n",
        "\n",
        "# Perform cross-validation\n",
        "for fold, (train_index, val_index) in enumerate(skf.split(X, y), 1):\n",
        "    X_val, y_val = X[val_index], y[val_index]\n",
        "\n",
        "    # Make predictions using the existing TPOT model\n",
        "    y_pred = tpot.predict(X_val)\n",
        "    y_pred_proba = tpot.predict_proba(X_val)[:, 1]\n",
        "\n",
        "    # Calculate scores\n",
        "    balanced_accuracy_scores.append(balanced_accuracy_score(y_val, y_pred))\n",
        "    roc_auc_scores.append(roc_auc_score(y_val, y_pred_proba))\n",
        "    average_precision_scores.append(average_precision_score(y_val, y_pred_proba))\n",
        "\n",
        "    print(f\"Fold {fold}:\")\n",
        "    print(f\"  Balanced Accuracy: {balanced_accuracy_scores[-1]:.4f}\")\n",
        "    print(f\"  ROC AUC: {roc_auc_scores[-1]:.4f}\")\n",
        "    print(f\"  Average Precision: {average_precision_scores[-1]:.4f}\")\n",
        "\n",
        "# Print the overall results\n",
        "print(\"\\nCross-validation results:\")\n",
        "print(f\"Balanced Accuracy: {np.mean(balanced_accuracy_scores):.4f} (+/- {np.std(balanced_accuracy_scores) * 2:.4f})\")\n",
        "print(f\"ROC AUC: {np.mean(roc_auc_scores):.4f} (+/- {np.std(roc_auc_scores) * 2:.4f})\")\n",
        "print(f\"Average Precision: {np.mean(average_precision_scores):.4f} (+/- {np.std(average_precision_scores) * 2:.4f})\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nmu5z7OVdQHA",
        "outputId": "8193a942-2d4f-45b6-d1dc-e0dfed3e1d6c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fold 1:\n",
            "  Balanced Accuracy: 0.9507\n",
            "  ROC AUC: 0.9996\n",
            "  Average Precision: 1.0000\n",
            "Fold 2:\n",
            "  Balanced Accuracy: 0.9472\n",
            "  ROC AUC: 0.9963\n",
            "  Average Precision: 1.0000\n",
            "Fold 3:\n",
            "  Balanced Accuracy: 0.9400\n",
            "  ROC AUC: 0.9982\n",
            "  Average Precision: 1.0000\n",
            "Fold 4:\n",
            "  Balanced Accuracy: 0.9457\n",
            "  ROC AUC: 1.0000\n",
            "  Average Precision: 1.0000\n",
            "Fold 5:\n",
            "  Balanced Accuracy: 0.9468\n",
            "  ROC AUC: 0.9956\n",
            "  Average Precision: 1.0000\n",
            "\n",
            "Cross-validation results:\n",
            "Balanced Accuracy: 0.9461 (+/- 0.0069)\n",
            "ROC AUC: 0.9979 (+/- 0.0035)\n",
            "Average Precision: 1.0000 (+/- 0.0000)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "\n",
        "# Assume these are saved from the training process\n",
        "saved_imputer = imputer  # The SimpleImputer used in training\n",
        "saved_scaler = scaler    # The StandardScaler used in training\n",
        "saved_pca = pca          # The TruncatedSVD (PCA) used in training\n",
        "\n",
        "# Vectorize the messages\n",
        "vector1 = vectorize_text(random_good_message)\n",
        "vector2 = vectorize_text(random_bad_message)\n",
        "\n",
        "print(\"Shape of vector1:\", vector1.shape)\n",
        "print(\"Shape of vector2:\", vector2.shape)\n",
        "\n",
        "# Combine the vectors\n",
        "X_new = np.vstack((vector1, vector2))\n",
        "\n",
        "print(\"Shape of X_new before preprocessing:\", X_new.shape)\n",
        "\n",
        "# Apply the saved preprocessing steps\n",
        "X_new = saved_imputer.transform(X_new)\n",
        "print(\"Shape after imputation:\", X_new.shape)\n",
        "\n",
        "X_new = saved_scaler.transform(X_new)\n",
        "print(\"Shape after scaling:\", X_new.shape)\n",
        "\n",
        "X_new = saved_pca.transform(X_new)\n",
        "print(\"Shape after PCA:\", X_new.shape)\n",
        "\n",
        "# Ensure X_new has exactly 100 features\n",
        "assert X_new.shape[1] == 100, f\"X_new has {X_new.shape[1]} features instead of 100\"\n",
        "\n",
        "# Make predictions using your TPOT model\n",
        "predictions = tpot.predict(X_new)\n",
        "probabilities = tpot.predict_proba(X_new)\n",
        "\n",
        "# Print results\n",
        "for i, (pred, prob) in enumerate(zip(predictions, probabilities)):\n",
        "    print(f\"Message {i+1}:\")\n",
        "    print(f\"Predicted class: {pred}\")\n",
        "    print(f\"Class probabilities: {prob}\")\n",
        "    print()\n",
        "\n",
        "# Print the shape of X_new for final confirmation\n",
        "print(f\"Final shape of X_new: {X_new.shape}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_AkVOj7ehjcQ",
        "outputId": "b4cd08a9-38e0-4f33-c910-e2cb43610837"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of vector1: (1, 30000)\n",
            "Shape of vector2: (1, 30000)\n",
            "Shape of X_new before preprocessing: (2, 30000)\n",
            "Shape after imputation: (2, 30000)\n",
            "Shape after scaling: (2, 30000)\n",
            "Shape after PCA: (2, 100)\n",
            "Message 1:\n",
            "Predicted class: 1\n",
            "Class probabilities: [0.186 0.814]\n",
            "\n",
            "Message 2:\n",
            "Predicted class: 0\n",
            "Class probabilities: [0.75 0.25]\n",
            "\n",
            "Final shape of X_new: (2, 100)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "import re\n",
        "from tabulate import tabulate\n",
        "\n",
        "# Assuming results_df is your DataFrame with all the run results\n",
        "\n",
        "def simplify_pipeline(pipeline_str):\n",
        "    # Extract the main classifier name\n",
        "    classifier = re.search(r\"(\\w+)\\(\", pipeline_str).group(1)\n",
        "    return classifier.lower()\n",
        "\n",
        "# Initialize a dictionary to store all pipeline statistics\n",
        "all_pipeline_stats = defaultdict(int)\n",
        "\n",
        "# Iterate through all runs and all evaluated pipelines\n",
        "for _, row in results_df.iterrows():\n",
        "    for pipeline_str in row['pipelines_tested']:\n",
        "        simple_pipeline = simplify_pipeline(pipeline_str)\n",
        "        all_pipeline_stats[simple_pipeline] += 1\n",
        "\n",
        "# Prepare the summary table\n",
        "summary_table = [[pipeline, count] for pipeline, count in all_pipeline_stats.items()]\n",
        "\n",
        "# Sort by count\n",
        "summary_table.sort(key=lambda x: -x[1])\n",
        "\n",
        "# Print the summary table\n",
        "print(\"\\nTop 10 Pipelines Across All Runs and Evaluations:\")\n",
        "print(tabulate(summary_table[:10], headers=['Pipeline', 'Count'], tablefmt='grid'))"
      ],
      "metadata": {
        "id": "9qalTrUsqpVD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building a Deep Learning NN classifier with Tpot (scikit Neural Network) - for an imbalanced dataset"
      ],
      "metadata": {
        "id": "PEWJBJqf15m-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# File name to search for\n",
        "file_name = \"lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered_vectors.parquet\"\n",
        "\n",
        "# Get the download URL of the specific file\n",
        "# download_url = get_specific_file_from_latest_release(github_token, repository_name, file_name)\n",
        "download_url = get_specific_file_from_tagged_release(github_token, repository_name, \"lab\", file_name)\n",
        "print(download_url)\n",
        "\n",
        "if download_url:\n",
        "    local_file_path = file_name\n",
        "    download_file(download_url, github_token, local_file_path)\n",
        "else:\n",
        "    print(\"File not found.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85,
          "referenced_widgets": [
            "8a807c2780cf4a65ad92dfa1ead0e439",
            "021bf8d789b74e3d899a99eeb82a0036",
            "f7aeed7c0692419591ca57bffee51739",
            "d7f2da7e3beb42dfaf31dc69eb7a597e",
            "c7b913c695fc448c8fc7b31f7098f636",
            "a59325353a1d4902b03f5a37c0293934",
            "6cd3d0fc462c4b14bbe6be92b8e6103e",
            "0cfdc6a2fa49499e9f0ce350b88dfbc4",
            "a9c70dae97424ca290a75e10d24baf52",
            "f0fc3fee01b04ffc922096ee18b6cb0f",
            "9142d6e9793648a9909e5ce3dabf415b"
          ]
        },
        "id": "SGH18PwBSdeE",
        "outputId": "6bb659aa-bdf4-49fc-ad4a-c9f8968ce3c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://api.github.com/repos/norandom/log2ml/releases/assets/182698628\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0.00/3.29M [00:00<?, ?iB/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8a807c2780cf4a65ad92dfa1ead0e439"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "File downloaded successfully and saved as lab_logs_blindtest_activity_sysmon_1000samples_july_28_2024_filtered_vectors.parquet\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Split data\n",
        "X = np.array(df['message_vector'].tolist())\n",
        "y = df['label_encoded'].values"
      ],
      "metadata": {
        "id": "kFKHmuKeWqbM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.metrics import make_scorer, confusion_matrix, recall_score\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from tpot import TPOTClassifier\n",
        "from collections import defaultdict\n",
        "import time\n",
        "import re\n",
        "import gc\n",
        "\n",
        "def print_step_info(X, y, step_name):\n",
        "    print(f\"\\n--- {step_name} ---\")\n",
        "    print(f\"X shape: {X.shape}\")\n",
        "    print(f\"y shape: {y.shape}\")\n",
        "    print(f\"Class distribution: {np.bincount(y)}\")\n",
        "\n",
        "def custom_scorer(y_true, y_pred):\n",
        "    cm = confusion_matrix(y_true, y_pred)\n",
        "    tn, fp, fn, tp = cm.ravel()\n",
        "    if (tp + fn) == 0 or (tn + fp) == 0:\n",
        "        return 0.0\n",
        "    recall = tp / (tp + fn + 1e-8)\n",
        "    specificity = tn / (tn + fp + 1e-8)\n",
        "    recall_weight = 1.0\n",
        "    specificity_weight = 2.0\n",
        "    score = (recall_weight * recall + specificity_weight * specificity) / (recall_weight + specificity_weight)\n",
        "    return score\n",
        "\n",
        "custom_scorer_obj = make_scorer(custom_scorer, greater_is_better=True)\n",
        "\n",
        "# Assuming df is already loaded and contains 'message_vector' and 'label' columns\n",
        "\n",
        "# Encode labels\n",
        "le = LabelEncoder()\n",
        "df['label_encoded'] = le.fit_transform(df['label'])\n",
        "\n",
        "# Split data\n",
        "X = np.array(df['message_vector'].tolist())\n",
        "y = df['label_encoded'].values\n",
        "\n",
        "print_step_info(X, y, \"Original Data\")\n",
        "\n",
        "# Preprocessing\n",
        "imputer = SimpleImputer(strategy='mean')\n",
        "scaler = StandardScaler()\n",
        "pca = TruncatedSVD(n_components=100, random_state=42)  # Reduce to 100 components\n",
        "\n",
        "X = imputer.fit_transform(X)\n",
        "X = scaler.fit_transform(X)\n",
        "X = pca.fit_transform(X)\n",
        "\n",
        "print_step_info(X, y, \"After Preprocessing\")\n",
        "\n",
        "# Split the data\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n",
        "print_step_info(X_train, y_train, \"Training Data\")\n",
        "\n",
        "# Resampling\n",
        "smote = SMOTE(sampling_strategy=0.5, random_state=42)\n",
        "rus = RandomUnderSampler(sampling_strategy=0.5, random_state=42)\n",
        "X_resampled, y_resampled = rus.fit_resample(*smote.fit_resample(X_train, y_train))\n",
        "print_step_info(X_resampled, y_resampled, \"Resampled Data\")\n",
        "\n",
        "# TPOT configuration for neural networks\n",
        "tpot_config = {\n",
        "    'sklearn.neural_network.MLPClassifier': {\n",
        "        'hidden_layer_sizes': [(50,), (100,), (50, 50), (100, 50)],\n",
        "        'activation': ['relu', 'tanh'],\n",
        "        'alpha': [0.0001, 0.001, 0.01],\n",
        "        'learning_rate': ['constant', 'adaptive'],\n",
        "    }\n",
        "}\n",
        "\n",
        "# Initialize results storage\n",
        "results = defaultdict(list)\n",
        "\n",
        "# Number of runs\n",
        "n_runs = 3\n",
        "\n",
        "# Initialize best_tpot and best_recall\n",
        "best_tpot = None\n",
        "best_recall = 0\n",
        "\n",
        "for run in range(n_runs):\n",
        "    print(f\"\\nStarting run {run + 1}/{n_runs}\")\n",
        "    start_time = time.time()\n",
        "\n",
        "    # Initialize TPOT\n",
        "    tpot = TPOTClassifier(\n",
        "        config_dict=tpot_config,\n",
        "        generations=5,\n",
        "        population_size=20,\n",
        "        verbosity=2,\n",
        "        scoring=custom_scorer_obj,\n",
        "        random_state=42 + run,\n",
        "        n_jobs=-1,\n",
        "        max_time_mins=120,\n",
        "        max_eval_time_mins=20\n",
        "    )\n",
        "\n",
        "    try:\n",
        "        # Fit TPOT\n",
        "        tpot.fit(X_resampled, y_resampled)\n",
        "\n",
        "        # Make predictions\n",
        "        y_pred = tpot.predict(X_test)\n",
        "\n",
        "        # Calculate recall score\n",
        "        recall = recall_score(y_test, y_pred, average='macro')\n",
        "\n",
        "        # Update best_tpot if this run has better recall score\n",
        "        if recall > best_recall:\n",
        "            best_recall = recall\n",
        "            best_tpot = tpot\n",
        "\n",
        "        # Get pipeline string\n",
        "        pipeline_str = str(tpot.fitted_pipeline_)\n",
        "\n",
        "        # Store results\n",
        "        results['run'].append(run + 1)\n",
        "        results['recall_score'].append(recall)\n",
        "        results['best_pipeline'].append(pipeline_str)\n",
        "        results['runtime'].append(time.time() - start_time)\n",
        "        results['pipelines_tested'].append(len(tpot.evaluated_individuals_))\n",
        "\n",
        "        print(f\"Run {run + 1} completed. Recall Score: {recall:.4f}, Pipelines tested: {len(tpot.evaluated_individuals_)}\")\n",
        "\n",
        "    except Exception as e:\n",
        "        print(f\"An error occurred during TPOT optimization: {str(e)}\")\n",
        "\n",
        "    # Clear memory\n",
        "    gc.collect()\n",
        "\n",
        "# Convert results to DataFrame\n",
        "results_df = pd.DataFrame(results)\n",
        "\n",
        "# Save results to CSV\n",
        "results_df.to_csv('tpot_nn_recall_results.csv', index=False)\n",
        "\n",
        "print(\"\\nResults saved to 'tpot_nn_recall_results.csv'\")\n",
        "\n",
        "# If best_tpot exists, export the best pipeline\n",
        "if best_tpot:\n",
        "    best_tpot.export('tpot_nn_best_pipeline.py')\n",
        "    print(\"Best pipeline exported to 'tpot_nn_best_pipeline.py'\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "873451c433c1441796e0f2c582b54edc",
            "12bd969e54e64b17b22957ca3da533fa",
            "4dd1615acf4645d7861226a3da60ae27",
            "b5c015fcd9bd4461885e5e5c7f49fa9a",
            "c46835dbd92c49d6aef630a6d74f8588",
            "8bf87a1e0740469cbea6f0c76dd37735",
            "cc4bbcf9f0c44cd69117dc6251954049",
            "b988186cafdf4da2a30bcff8a6a00f70",
            "f998e026eba0488288515e8b69eacc96",
            "0df60724539b4708a9323dbb479fa044",
            "dad976092fff4859aaf73eb9d85aa01b",
            "4b8ebee65b2e4789bce617bedb691cb7",
            "a2a4e23c74fb4892be3f9edd122efd35",
            "1abae52fd84841759c034f9fb78c65a3",
            "36359ad021e64e6eb733ae740d987e68",
            "80d36d8a650f42d8a990e04014e937a1",
            "751c1855804a48ff9f0d6496a5e29506",
            "4749064df71f48f4b947ba208acbac26",
            "14db8797959a4253a36dd334f4826a00",
            "94e3a14430f64fa9ab318f0e6c37a883",
            "9b974c27ffe7444ab7f9683167a6988c",
            "e13925b2a40c4dcf993c62f64d332272",
            "d143776d4b464a648b17ae8e280ecc39",
            "16a6911e48b142cf83401adb3788792f",
            "68f7ddcbed594c8c8a511c20be8d4acd",
            "4c3ff39abc4b4f0baeee0538ce75d76f",
            "1f8d747cea354729af6b532fb2a23a97",
            "f634d0b3e4c34f058ab52077ccb97407",
            "5df15aaaa49b425e827b57a427479f89",
            "11ac4cb2fd7641e98779b027d47f2696",
            "d7a0b39588cb421492dde89492430a84",
            "fe991ac34e224d55b9c27ca1bad2808b",
            "ba7425e6aedf46f5adce1a7e8e2a7c4d"
          ]
        },
        "id": "iyExRJRjsDBO",
        "outputId": "48d4a650-08a3-4766-c47b-058b16b49222"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "--- Original Data ---\n",
            "X shape: (13455, 30000)\n",
            "y shape: (13455,)\n",
            "Class distribution: [  114 13341]\n",
            "\n",
            "--- After Preprocessing ---\n",
            "X shape: (13455, 100)\n",
            "y shape: (13455,)\n",
            "Class distribution: [  114 13341]\n",
            "\n",
            "--- Training Data ---\n",
            "X shape: (10764, 100)\n",
            "y shape: (10764,)\n",
            "Class distribution: [   91 10673]\n",
            "\n",
            "--- Resampled Data ---\n",
            "X shape: (16008, 100)\n",
            "y shape: (16008,)\n",
            "Class distribution: [ 5336 10672]\n",
            "\n",
            "Starting run 1/3\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "873451c433c1441796e0f2c582b54edc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Optimization Progress:   0%|          | 0/20 [00:00<?, ?pipeline/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Generation 1 - Current best internal CV score: 0.9990629684585782\n",
            "\n",
            "Generation 2 - Current best internal CV score: 0.9990629830910078\n",
            "\n",
            "Generation 3 - Current best internal CV score: 0.9990629830910078\n",
            "\n",
            "Generation 4 - Current best internal CV score: 0.9990629830910078\n",
            "\n",
            "Generation 5 - Current best internal CV score: 0.9990629977234378\n",
            "\n",
            "Best pipeline: MLPClassifier(MLPClassifier(input_matrix, activation=relu, alpha=0.001, hidden_layer_sizes=(50, 50), learning_rate=adaptive), activation=relu, alpha=0.0001, hidden_layer_sizes=(100, 50), learning_rate=adaptive)\n",
            "Run 1 completed. Recall Score: 0.8250, Pipelines tested: 89\n",
            "\n",
            "Starting run 2/3\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4b8ebee65b2e4789bce617bedb691cb7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Optimization Progress:   0%|          | 0/20 [00:00<?, ?pipeline/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "Generation 1 - Current best internal CV score: 0.9992815476935497\n",
            "\n",
            "Generation 2 - Current best internal CV score: 0.9992815476935497\n",
            "\n",
            "Generation 3 - Current best internal CV score: 0.9992815476935497\n",
            "\n",
            "Generation 4 - Current best internal CV score: 0.9992815476935497\n",
            "\n",
            "Generation 5 - Current best internal CV score: 0.9993127732985456\n",
            "\n",
            "Best pipeline: MLPClassifier(MLPClassifier(input_matrix, activation=tanh, alpha=0.01, hidden_layer_sizes=(100, 50), learning_rate=adaptive), activation=relu, alpha=0.01, hidden_layer_sizes=(100, 50), learning_rate=adaptive)\n",
            "Run 2 completed. Recall Score: 0.9119, Pipelines tested: 95\n",
            "\n",
            "Starting run 3/3\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d143776d4b464a648b17ae8e280ecc39",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Optimization Progress:   0%|          | 0/20 [00:00<?, ?pipeline/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 1 - Current best internal CV score: 0.999062924561289\n",
            "\n",
            "Generation 2 - Current best internal CV score: 0.9991254196685702\n",
            "\n",
            "Generation 3 - Current best internal CV score: 0.9991254196685702\n",
            "\n",
            "Generation 4 - Current best internal CV score: 0.9991254196685702\n",
            "\n",
            "Generation 5 - Current best internal CV score: 0.9991254343009999\n",
            "\n",
            "Best pipeline: MLPClassifier(MLPClassifier(input_matrix, activation=tanh, alpha=0.0001, hidden_layer_sizes=(100,), learning_rate=constant), activation=relu, alpha=0.0001, hidden_layer_sizes=(100, 50), learning_rate=adaptive)\n",
            "Run 3 completed. Recall Score: 0.8025, Pipelines tested: 95\n",
            "\n",
            "Results saved to 'tpot_nn_recall_results.csv'\n",
            "Best pipeline exported to 'tpot_nn_best_pipeline.py'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from scipy import stats\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Assuming results_df is already loaded from 'tpot_nn_recall_results.csv'\n",
        "# If not, uncomment the following line:\n",
        "# results_df = pd.read_csv('tpot_nn_recall_results.csv')\n",
        "\n",
        "# Print summary statistics\n",
        "print(\"\\nSummary Statistics:\")\n",
        "print(results_df.describe())\n",
        "\n",
        "# Calculate confidence intervals for recall scores\n",
        "confidence_level = 0.95\n",
        "recall_scores = results_df['recall_score']\n",
        "recall_mean = np.mean(recall_scores)\n",
        "recall_std = np.std(recall_scores, ddof=1)\n",
        "recall_se = recall_std / np.sqrt(len(recall_scores))\n",
        "recall_ci = stats.t.interval(confidence_level, len(recall_scores)-1, loc=recall_mean, scale=recall_se)\n",
        "\n",
        "print(f\"\\nRecall Score Analysis:\")\n",
        "print(f\"  Mean: {recall_mean:.4f}\")\n",
        "print(f\"  Standard Deviation: {recall_std:.4f}\")\n",
        "print(f\"  {confidence_level*100}% Confidence Interval: ({recall_ci[0]:.4f}, {recall_ci[1]:.4f})\")\n",
        "\n",
        "# Perform t-test to check if recall scores are significantly different from a baseline\n",
        "baseline_recall = 0.5  # You can adjust this value based on your expectations\n",
        "t_stat, p_value = stats.ttest_1samp(recall_scores, baseline_recall)\n",
        "print(f\"\\nOne-sample t-test (comparing to baseline recall of {baseline_recall}):\")\n",
        "print(f\"  t-statistic: {t_stat:.4f}\")\n",
        "print(f\"  p-value: {p_value:.4f}\")\n",
        "\n",
        "# Check for normality of recall scores\n",
        "_, normality_p_value = stats.shapiro(recall_scores)\n",
        "print(f\"\\nShapiro-Wilk test for normality of recall scores:\")\n",
        "print(f\"  p-value: {normality_p_value:.4f}\")\n",
        "print(f\"  {'Recall scores are normally distributed' if normality_p_value > 0.05 else 'Recall scores are not normally distributed'}\")\n",
        "\n",
        "# Calculate effect size (Cohen's d) for recall scores compared to baseline\n",
        "effect_size = (recall_mean - baseline_recall) / recall_std\n",
        "print(f\"\\nEffect size (Cohen's d) compared to baseline: {effect_size:.4f}\")\n",
        "\n",
        "# Plot recall scores\n",
        "plt.figure(figsize=(10, 6))\n",
        "plt.plot(results_df['run'], results_df['recall_score'], 'bo-')\n",
        "plt.title('Recall Scores Across Runs')\n",
        "plt.xlabel('Run')\n",
        "plt.ylabel('Recall Score')\n",
        "plt.grid(True)\n",
        "plt.tight_layout()\n",
        "plt.show()\n",
        "\n",
        "# Runtime analysis\n",
        "print(\"\\nRuntime Analysis:\")\n",
        "print(f\"  Total Runtime: {results_df['runtime'].sum():.2f} seconds\")\n",
        "print(f\"  Average Runtime per Run: {results_df['runtime'].mean():.2f} seconds\")\n",
        "\n",
        "# Pipelines tested analysis\n",
        "print(\"\\nPipelines Tested Analysis:\")\n",
        "print(f\"  Total Pipelines Tested: {results_df['pipelines_tested'].sum()}\")\n",
        "print(f\"  Average Pipelines per Run: {results_df['pipelines_tested'].mean():.2f}\")\n",
        "\n",
        "# Best pipeline\n",
        "best_run = results_df.loc[results_df['recall_score'].idxmax()]\n",
        "print(f\"\\nBest Run (Run {best_run['run']}):\")\n",
        "print(f\"  Recall Score: {best_run['recall_score']:.4f}\")\n",
        "print(f\"  Runtime: {best_run['runtime']:.2f} seconds\")\n",
        "print(f\"  Pipelines Tested: {best_run['pipelines_tested']}\")\n",
        "print(f\"  Best Pipeline:\\n{best_run['best_pipeline']}\")\n",
        "\n",
        "# Plot runtime vs recall score\n",
        "plt.figure(figsize=(10, 6))\n",
        "plt.scatter(results_df['runtime'], results_df['recall_score'])\n",
        "plt.title('Runtime vs Recall Score')\n",
        "plt.xlabel('Runtime (seconds)')\n",
        "plt.ylabel('Recall Score')\n",
        "plt.grid(True)\n",
        "plt.tight_layout()\n",
        "plt.show()\n",
        "\n",
        "# Analyze the relationship between runtime and recall score\n",
        "runtime_recall_corr, _ = stats.pearsonr(results_df['runtime'], results_df['recall_score'])\n",
        "print(f\"\\nCorrelation between runtime and recall score: {runtime_recall_corr:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "s7pRao7WxkB8",
        "outputId": "eadc42dd-2673-4d07-f979-cb49327f30ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Summary Statistics:\n",
            "       run  recall_score     runtime  pipelines_tested\n",
            "count  3.0      3.000000    3.000000          3.000000\n",
            "mean   2.0      0.846452  482.763048         93.000000\n",
            "std    1.0      0.057801   26.729168          3.464102\n",
            "min    1.0      0.802474  459.580693         89.000000\n",
            "25%    1.5      0.813718  468.144446         92.000000\n",
            "50%    2.0      0.824963  476.708200         95.000000\n",
            "75%    2.5      0.868441  494.354225         95.000000\n",
            "max    3.0      0.911919  512.000251         95.000000\n",
            "\n",
            "Recall Score Analysis:\n",
            "  Mean: 0.8465\n",
            "  Standard Deviation: 0.0578\n",
            "  95.0% Confidence Interval: (0.7029, 0.9900)\n",
            "\n",
            "One-sample t-test (comparing to baseline recall of 0.5):\n",
            "  t-statistic: 10.3818\n",
            "  p-value: 0.0092\n",
            "\n",
            "Shapiro-Wilk test for normality of recall scores:\n",
            "  p-value: 0.3739\n",
            "  Recall scores are normally distributed\n",
            "\n",
            "Effect size (Cohen's d) compared to baseline: 5.9939\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 48952 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Runtime Analysis:\n",
            "  Total Runtime: 1448.29 seconds\n",
            "  Average Runtime per Run: 482.76 seconds\n",
            "\n",
            "Pipelines Tested Analysis:\n",
            "  Total Pipelines Tested: 279\n",
            "  Average Pipelines per Run: 93.00\n",
            "\n",
            "Best Run (Run 2):\n",
            "  Recall Score: 0.9119\n",
            "  Runtime: 512.00 seconds\n",
            "  Pipelines Tested: 95\n",
            "  Best Pipeline:\n",
            "Pipeline(steps=[('stackingestimator',\n",
            "                 StackingEstimator(estimator=MLPClassifier(activation='tanh',\n",
            "                                                           alpha=0.01,\n",
            "                                                           hidden_layer_sizes=(100,\n",
            "                                                                               50),\n",
            "                                                           learning_rate='adaptive',\n",
            "                                                           random_state=43))),\n",
            "                ('mlpclassifier',\n",
            "                 MLPClassifier(alpha=0.01, hidden_layer_sizes=(100, 50),\n",
            "                               learning_rate='adaptive', random_state=43))])\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 29300 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Correlation between runtime and recall score: 0.8669\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "K-pwEYJV-uZn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions\n",
        "y_pred = best_tpot.predict(X_test)\n",
        "\n",
        "# Create the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "\n",
        "# Create a ConfusionMatrixDisplay object\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['negative', 'positive'])\n",
        "\n",
        "# Plot the confusion matrix\n",
        "plt.figure(figsize=(10, 7))\n",
        "disp.plot(cmap='Blues', values_format='d')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.show()\n",
        "\n",
        "# Print the confusion matrix\n",
        "print(\"Confusion Matrix:\")\n",
        "print(cm)\n",
        "\n",
        "# Calculate and print additional metrics\n",
        "tn, fp, fn, tp = cm.ravel()\n",
        "precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n",
        "recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n",
        "f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
        "\n",
        "print(f\"\\nPrecision: {precision:.4f}\")\n",
        "print(f\"Recall: {recall:.4f}\")\n",
        "print(f\"F1-score: {f1_score:.4f}\")\n",
        "\n",
        "# Additional metrics from the custom scorer\n",
        "specificity = tn / (tn + fp) if (tn + fp) > 0 else 0\n",
        "custom_score = (recall + 2 * specificity) / 3  # As per your custom scorer\n",
        "\n",
        "print(f\"Specificity: {specificity:.4f}\")\n",
        "print(f\"Custom Score: {custom_score:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 653
        },
        "id": "cmtpHeIzxk7f",
        "outputId": "3ef42227-4089-4d84-a3fc-de50ab86cc99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "[encoded data, 28200 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Confusion Matrix:\n",
            "[[  19    4]\n",
            " [   6 2662]]\n",
            "\n",
            "Precision: 0.9985\n",
            "Recall: 0.9978\n",
            "F1-score: 0.9981\n",
            "Specificity: 0.8261\n",
            "Custom Score: 0.8833\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import re\n",
        "\n",
        "# Assuming results_df is already loaded from 'tpot_nn_recall_results.csv'\n",
        "# If not, uncomment the following line:\n",
        "# results_df = pd.read_csv('tpot_nn_recall_results.csv')\n",
        "\n",
        "def extract_pipeline_components(pipeline_str):\n",
        "    # Extract all components from the pipeline string, excluding 'Pipeline'\n",
        "    components = re.findall(r'(\\w+)\\(', pipeline_str)\n",
        "    return ', '.join([comp for comp in components if comp != 'Pipeline'])\n",
        "\n",
        "# Extract pipeline components\n",
        "results_df['pipeline_components'] = results_df['best_pipeline'].apply(extract_pipeline_components)\n",
        "\n",
        "# Create a table of pipeline components and their frequencies\n",
        "component_counts = results_df['pipeline_components'].value_counts().reset_index()\n",
        "component_counts.columns = ['Pipeline Components', 'Frequency']\n",
        "component_counts['Percentage'] = component_counts['Frequency'] / len(results_df) * 100\n",
        "\n",
        "print(\"Pipeline Components Frequency Table:\")\n",
        "print(component_counts.to_string(index=False))\n",
        "\n",
        "# Visualize pipeline components frequency\n",
        "plt.figure(figsize=(12, 6))\n",
        "component_counts.plot(kind='bar', x='Pipeline Components', y='Frequency')\n",
        "plt.title('Frequency of Pipeline Components in Best Pipelines')\n",
        "plt.xlabel('Pipeline Components')\n",
        "plt.ylabel('Frequency')\n",
        "plt.xticks(rotation=45, ha='right')\n",
        "plt.tight_layout()\n",
        "plt.show()\n",
        "\n",
        "# Find the best performing pipeline components\n",
        "best_components = results_df.loc[results_df['recall_score'].idxmax(), 'pipeline_components']\n",
        "print(f\"\\nComponents of the best performing pipeline: {best_components}\")\n",
        "\n",
        "# Analyze individual components\n",
        "all_components = [comp.strip() for comps in results_df['pipeline_components'].str.split(',') for comp in comps]\n",
        "component_freq = pd.Series(all_components).value_counts()\n",
        "\n",
        "print(\"\\nIndividual Component Frequency:\")\n",
        "print(component_freq)\n",
        "\n",
        "# Visualize individual component frequency\n",
        "plt.figure(figsize=(10, 6))\n",
        "component_freq.plot(kind='bar')\n",
        "plt.title('Frequency of Individual Components in Best Pipelines')\n",
        "plt.xlabel('Component')\n",
        "plt.ylabel('Frequency')\n",
        "plt.xticks(rotation=45, ha='right')\n",
        "plt.tight_layout()\n",
        "plt.show()\n",
        "\n",
        "# Additional analysis: Component combination performance\n",
        "results_df['mean_recall'] = results_df.groupby('pipeline_components')['recall_score'].transform('mean')\n",
        "best_combo = results_df.loc[results_df['mean_recall'].idxmax(), 'pipeline_components']\n",
        "best_combo_recall = results_df.loc[results_df['mean_recall'].idxmax(), 'mean_recall']\n",
        "\n",
        "print(f\"\\nBest performing component combination: {best_combo}\")\n",
        "print(f\"Mean recall score: {best_combo_recall:.4f}\")\n",
        "\n",
        "# [Rest of the previous analysis code remains the same]\n",
        "\n",
        "# ... [Keep all the previous analysis code] ..."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "lUqkJTy-0SCE",
        "outputId": "d343d5d2-741a-4c32-f904-7f4281bf258b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pipeline Components Frequency Table:\n",
            "                            Pipeline Components  Frequency  Percentage\n",
            "StackingEstimator, MLPClassifier, MLPClassifier          3       100.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x600 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "[encoded data, 36968 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Components of the best performing pipeline: StackingEstimator, MLPClassifier, MLPClassifier\n",
            "\n",
            "Individual Component Frequency:\n",
            "MLPClassifier        6\n",
            "StackingEstimator    3\n",
            "Name: count, dtype: int64\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x600 with 1 Axes>"
            ],
            "image/png": "[encoded data, 33872 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Best performing component combination: StackingEstimator, MLPClassifier, MLPClassifier\n",
            "Mean recall score: 0.8465\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "iaMlGpRsxk_F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "# Find the best run (highest F1 score)\n",
        "best_run_index = results_df['f1_score'].idxmax()\n",
        "best_pipeline_str = results_df.loc[best_run_index, 'best_pipeline']\n",
        "\n",
        "# Re-fit the best pipeline on the entire dataset\n",
        "X_train_full, X_test_full, y_train_full, y_test_full = train_test_split(X_pca, y_encoded, test_size=0.2, random_state=42, stratify=y_encoded)\n",
        "best_model = tpot.fitted_pipeline_\n",
        "\n",
        "# Save the model using joblib\n",
        "model_filename = 'best_tpot_nn_model.joblib'\n",
        "joblib.dump(best_model, model_filename)\n",
        "\n",
        "print(f\"\\nBest model exported as joblib: {model_filename}\")\n",
        "\n",
        "# You can also save the best pipeline as a Python script\n",
        "python_filename = 'best_tpot_nn_pipeline.py'\n",
        "best_pipeline_script = tpot.export(python_filename)\n",
        "\n",
        "print(f\"Best pipeline saved as Python script: {python_filename}\")\n",
        "\n",
        "# Print the best pipeline\n",
        "print(f\"\\nBest Pipeline:\\n{best_pipeline_str}\")\n",
        "\n",
        "# Verify the saved model\n",
        "try:\n",
        "    loaded_model = joblib.load(model_filename)\n",
        "    y_pred = loaded_model.predict(X_test_full)\n",
        "    f1 = f1_score(y_test_full, y_pred, average='weighted')\n",
        "    print(f\"\\nLoaded model verified. F1 Score: {f1:.4f}\")\n",
        "except Exception as e:\n",
        "    print(f\"\\nError verifying saved model: {str(e)}\")\n",
        "\n",
        "# Print information about how to use the saved model\n",
        "print(\"\\nTo use the saved model in the future:\")\n",
        "print(\"1. Load the model: loaded_model = joblib.load('best_tpot_nn_model.joblib')\")\n",
        "print(\"2. Make predictions: predictions = loaded_model.predict(X)\")\n",
        "print(\"Note: Make sure to preprocess your input data (X) the same way as during training.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tQ06A8TMgdKw",
        "outputId": "8e9cbfe1-8098-437a-c184-9172d0284db7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Best model exported as joblib: best_tpot_nn_model.joblib\n",
            "Best pipeline saved as Python script: best_tpot_nn_pipeline.py\n",
            "\n",
            "Best Pipeline:\n",
            "Pipeline(steps=[('gaussiannb', GaussianNB())])\n",
            "\n",
            "Loaded model verified. F1 Score: 0.9988\n",
            "\n",
            "To use the saved model in the future:\n",
            "1. Load the model: loaded_model = joblib.load('best_tpot_nn_model.joblib')\n",
            "2. Make predictions: predictions = loaded_model.predict(X)\n",
            "Note: Make sure to preprocess your input data (X) the same way as during training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Attempt: Building a GPU-accelerated cuML classifier with Tpot - for an imbalanced dataset"
      ],
      "metadata": {
        "id": "QJQXi06FdUDN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import make_scorer, confusion_matrix, recall_score, accuracy_score, f1_score\n",
        "from sklearn.preprocessing import LabelEncoder, QuantileTransformer\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from cuml.ensemble import RandomForestClassifier\n",
        "from cuml.svm import SVC\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from tpot import TPOTClassifier\n",
        "import time\n",
        "import gc\n",
        "import warnings\n",
        "\n",
        "# Suppress warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "def custom_scorer(y_true, y_pred):\n",
        "    cm = confusion_matrix(y_true, y_pred)\n",
        "    tn, fp, fn, tp = cm.ravel()\n",
        "    if (tp + fn) == 0 or (tn + fp) == 0:\n",
        "        return 0.0\n",
        "    recall = tp / (tp + fn + 1e-8)\n",
        "    specificity = tn / (tn + fp + 1e-8)\n",
        "    recall_weight = 1.0\n",
        "    specificity_weight = 2.0\n",
        "    score = (recall_weight * recall + specificity_weight * specificity) / (recall_weight + specificity_weight)\n",
        "    return score\n",
        "\n",
        "custom_scorer_obj = make_scorer(custom_scorer, greater_is_better=True)\n",
        "\n",
        "# Assuming df is already loaded and contains 'message_vector' and 'label' columns\n",
        "\n",
        "# Prepare data\n",
        "X = np.array(df['message_vector'].tolist(), dtype='float32')\n",
        "y = df['label'].values\n",
        "\n",
        "# Ensure y is numeric\n",
        "le = LabelEncoder()\n",
        "y = le.fit_transform(y)\n",
        "\n",
        "# Preprocessing\n",
        "qt = QuantileTransformer(n_quantiles=1000, output_distribution='normal')\n",
        "pca = TruncatedSVD(n_components=min(100, X.shape[1] - 1), random_state=42)\n",
        "X = qt.fit_transform(X)\n",
        "X = pca.fit_transform(X)\n",
        "\n",
        "# Ensure X is float32 (required for cuML)\n",
        "X = X.astype(np.float32)\n",
        "\n",
        "# Split the data\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\n",
        "\n",
        "# Resampling\n",
        "smote = SMOTE(sampling_strategy=0.5, random_state=42)\n",
        "rus = RandomUnderSampler(sampling_strategy=0.5, random_state=42)\n",
        "X_train, y_train = rus.fit_resample(*smote.fit_resample(X_train, y_train))\n",
        "\n",
        "# Ensure y_train is int32 (required for cuML)\n",
        "y_train = y_train.astype(np.int32)\n",
        "\n",
        "# Custom configuration for TPOT\n",
        "tpot_config = {\n",
        "    'cuml.ensemble.RandomForestClassifier': {\n",
        "        'n_estimators': [100, 200, 500],\n",
        "        'max_depth': [10, 20, 30, None],\n",
        "        'min_samples_split': [2, 5, 10],\n",
        "        'min_samples_leaf': [1, 2, 4],\n",
        "        'max_features': ['sqrt', 'log2', 0.1, 0.3, 0.5],\n",
        "        'class_weight': [None, 'balanced', 'balanced_subsample'],\n",
        "        # 'criterion': ['gini', 'entropy'],\n",
        "        'bootstrap': [True, False]\n",
        "    }\n",
        "}\n",
        "\n",
        "# Initialize and run TPOT\n",
        "tpot = TPOTClassifier(\n",
        "    generations=5,\n",
        "    population_size=20,\n",
        "    cv=5,\n",
        "    scoring=custom_scorer_obj,\n",
        "    random_state=42,\n",
        "    config_dict=tpot_config,\n",
        "    verbosity=2,\n",
        "    n_jobs=1,\n",
        "    max_time_mins=60,\n",
        "    max_eval_time_mins=5\n",
        ")\n",
        "\n",
        "start_time = time.time()\n",
        "tpot.fit(X_train, y_train)\n",
        "print(f\"TPOT optimization completed in {time.time() - start_time:.2f} seconds\")\n",
        "\n",
        "# Evaluate the best pipeline\n",
        "y_pred = tpot.predict(X_test)\n",
        "recall = recall_score(y_test, y_pred, average='macro')\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "tn, fp, fn, tp = cm.ravel()\n",
        "specificity = tn / (tn + fp + 1e-8)\n",
        "accuracy = accuracy_score(y_test, y_pred)\n",
        "f1 = f1_score(y_test, y_pred, average='weighted')\n",
        "custom_score = custom_scorer(y_test, y_pred)\n",
        "\n",
        "print(f\"Best pipeline: {tpot.fitted_pipeline_}\")\n",
        "print(f\"Recall Score: {recall:.4f}\")\n",
        "print(f\"Specificity: {specificity:.4f}\")\n",
        "print(f\"Accuracy: {accuracy:.4f}\")\n",
        "print(f\"F1 Score: {f1:.4f}\")\n",
        "print(f\"Custom Score: {custom_score:.4f}\")\n",
        "\n",
        "print(\"Confusion Matrix:\")\n",
        "print(cm)\n",
        "\n",
        "# Export the best pipeline\n",
        "tpot.export('tpot_best_pipeline.py')\n",
        "print(\"Best pipeline exported to 'tpot_best_pipeline.py'\")\n",
        "\n",
        "gc.collect()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "a720f2592e3d4bef98a4c6f5077f277f",
            "d2ae2ec449c14266b009f213df117da2",
            "979b4554ad0b474b883040d9bbd51274",
            "be36aefa1b3e4ea6a776e02c082c9958",
            "7fc5e7ea894245579a1abdeed4492954",
            "88b18cc3b9cf4eaa8edc5b16aaf28821",
            "d5935c9201ed471ab2385df6e607380c",
            "4aa0b28b2aa44951aad3bfe37fa20051",
            "43490d899aed4338823374634d05c9f1",
            "512a81b06bd54811bd8b9f80958fa7a4",
            "201d44e85f374a06929ae8db5851d61b"
          ]
        },
        "id": "vD-eHcZD-wdY",
        "outputId": "afd6d273-d0bc-4353-cb2a-b2d6ef458cd1"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Optimization Progress:   0%|          | 0/20 [00:00<?, ?pipeline/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a720f2592e3d4bef98a4c6f5077f277f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 1 - Current best internal CV score: 0.9999062792799144\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 2 - Current best internal CV score: 0.9999062792799144\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 3 - Current best internal CV score: 0.9999375195173401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 4 - Current best internal CV score: 0.9999375195173401\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n",
            "Exception ignored in: <cyfunction RandomForestClassifier.__del__ at 0x7ac2ebd6e810>\n",
            "Traceback (most recent call last):\n",
            "  File \"randomforestclassifier.pyx\", line 314, in cuml.ensemble.randomforestclassifier.RandomForestClassifier.__del__\n",
            "  File \"randomforestclassifier.pyx\", line 318, in cuml.ensemble.randomforestclassifier.RandomForestClassifier._reset_forest_data\n",
            "  File \"base.pyx\", line 330, in cuml.internals.base.Base.__getattr__\n",
            "AttributeError: rf_forest\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Generation 5 - Current best internal CV score: 0.9999375195173401\n",
            "\n",
            "Best pipeline: RandomForestClassifier(CombineDFs(input_matrix, input_matrix), bootstrap=False, class_weight=None, max_depth=30, max_features=sqrt, min_samples_leaf=1, min_samples_split=5, n_estimators=200)\n",
            "TPOT optimization completed in 493.37 seconds\n",
            "Best pipeline: Pipeline(steps=[('featureunion',\n",
            "                 FeatureUnion(transformer_list=[('functiontransformer-1',\n",
            "                                                 FunctionTransformer(func=<function copy at 0x7acc3db1f520>)),\n",
            "                                                ('functiontransformer-2',\n",
            "                                                 FunctionTransformer(func=<function copy at 0x7acc3db1f520>))])),\n",
            "                ('randomforestclassifier', RandomForestClassifier())])\n",
            "Recall Score: 0.5652\n",
            "Specificity: 0.1304\n",
            "Accuracy: 0.9926\n",
            "F1 Score: 0.9897\n",
            "Custom Score: 0.4203\n",
            "Confusion Matrix:\n",
            "[[   3   20]\n",
            " [   0 2668]]\n",
            "Best pipeline exported to 'tpot_best_pipeline.py'\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2071"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Make predictions\n",
        "y_pred = tpot.predict(X_test)\n",
        "\n",
        "# Create the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "\n",
        "# Create a ConfusionMatrixDisplay object\n",
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['negative', 'positive'])\n",
        "\n",
        "# Plot the confusion matrix\n",
        "plt.figure(figsize=(10, 7))\n",
        "disp.plot(cmap='Blues', values_format='d')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.show()\n",
        "\n",
        "# Print the confusion matrix\n",
        "print(\"Confusion Matrix:\")\n",
        "print(cm)\n",
        "\n",
        "# Calculate and print additional metrics\n",
        "tn, fp, fn, tp = cm.ravel()\n",
        "precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n",
        "recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n",
        "f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
        "\n",
        "print(f\"\\nPrecision: {precision:.4f}\")\n",
        "print(f\"Recall: {recall:.4f}\")\n",
        "print(f\"F1-score: {f1_score:.4f}\")\n",
        "\n",
        "# Additional metrics from the custom scorer\n",
        "specificity = tn / (tn + fp) if (tn + fp) > 0 else 0\n",
        "custom_score = (recall + 2 * specificity) / 3  # As per your custom scorer\n",
        "\n",
        "print(f\"Specificity: {specificity:.4f}\")\n",
        "print(f\"Custom Score: {custom_score:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 653
        },
        "id": "lVua-bgO_9T8",
        "outputId": "67a8ddee-ab06-4b6e-a946-9ccb175b7401"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1000x700 with 0 Axes>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "[encoded data, 28824 characters]\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Confusion Matrix:\n",
            "[[   3   20]\n",
            " [   0 2668]]\n",
            "\n",
            "Precision: 0.9926\n",
            "Recall: 1.0000\n",
            "F1-score: 0.9963\n",
            "Specificity: 0.1304\n",
            "Custom Score: 0.4203\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "\n",
        "# Assume these are saved from the training process\n",
        "# saved_imputer = imputer  # The SimpleImputer used in training\n",
        "\n",
        "\n",
        "# Vectorize the messages\n",
        "vector1 = vectorize_text(random_good_message)\n",
        "vector2 = vectorize_text(random_bad_message)\n",
        "\n",
        "print(\"Shape of vector1:\", vector1.shape)\n",
        "print(\"Shape of vector2:\", vector2.shape)\n",
        "\n",
        "# Combine the vectors\n",
        "X_new = np.vstack((vector1, vector2))\n",
        "\n",
        "print(\"Shape of X_new before preprocessing:\", X_new.shape)\n",
        "\n",
        "\n",
        "X_new = qt.transform(X_new)\n",
        "print(\"Shape after scaling:\", X_new.shape)\n",
        "\n",
        "X_new = pca.transform(X_new)\n",
        "print(\"Shape after PCA:\", X_new.shape)\n",
        "\n",
        "# Ensure X_new has exactly 100 features\n",
        "assert X_new.shape[1] == 100, f\"X_new has {X_new.shape[1]} features instead of 100\"\n",
        "\n",
        "# Make predictions using your TPOT model\n",
        "predictions = tpot.predict(X_new)\n",
        "probabilities = tpot.predict_proba(X_new)\n",
        "\n",
        "print(random_bad_message)\n",
        "\n",
        "# Print results\n",
        "for i, (pred, prob) in enumerate(zip(predictions, probabilities)):\n",
        "    print(f\"Message {i+1}:\")\n",
        "    print(f\"Predicted class: {pred}\")\n",
        "    print(f\"Class probabilities: {prob}\")\n",
        "    print()\n",
        "\n",
        "# Print the shape of X_new for final confirmation\n",
        "print(f\"Final shape of X_new: {X_new.shape}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PTB4jZoZBDQU",
        "outputId": "cd93f788-e7b0-4b9f-cb33-33d5f617b933"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shape of vector1: (1, 30000)\n",
            "Shape of vector2: (1, 30000)\n",
            "Shape of X_new before preprocessing: (2, 30000)\n",
            "Shape after scaling: (2, 30000)\n",
            "Shape after PCA: (2, 100)\n",
            "File created:\n",
            "RuleName: EXE\n",
            "UtcTime: 2024-07-28 15:53:22.557\n",
            "ProcessGuid: {18e8265a-696d-66a6-7705-000000004400}\n",
            "ProcessId: 10624\n",
            "Image: C:\\Program Files\\Microsoft Office\\Root\\Office16\\EXCEL.EXE\n",
            "TargetFilename: C:\\Users\\student\\AppData\\Local\\Temp\\file.exe\n",
            "CreationUtcTime: 2024-07-23 14:24:50.520\n",
            "Message 1:\n",
            "Predicted class: 1.0\n",
            "Class probabilities: [0.12708335 0.8729167 ]\n",
            "\n",
            "Message 2:\n",
            "Predicted class: 1.0\n",
            "Class probabilities: [0.12375 0.87625]\n",
            "\n",
            "Final shape of X_new: (2, 100)\n"
          ]
        }
      ]
    }
  ]
}