NER_data_augmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NER_data_augmentation.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMEwMTGA/k2mJVOfIugC8i+",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "67eed08b809f49cdbde2a7adfa384826": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c0dbb696d1a3491fac930edd6c2edba1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4e9cfe21abae410f9550e15cce5b12ae",
              "IPY_MODEL_9e003f0e920749ba932db90a54efcad5"
            ]
          }
        },
        "c0dbb696d1a3491fac930edd6c2edba1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e9cfe21abae410f9550e15cce5b12ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3315bc7ca998472d97c7e321ce7e23c1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 480,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 480,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4c5b63100a1741fdb2ac079c7042b583"
          }
        },
        "9e003f0e920749ba932db90a54efcad5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_879ac1a5bc19487c9bafe1ab2eff8660",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 480/480 [00:01&lt;00:00, 244B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f9475b046d642f8b5fe6631bac779b3"
          }
        },
        "3315bc7ca998472d97c7e321ce7e23c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4c5b63100a1741fdb2ac079c7042b583": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "879ac1a5bc19487c9bafe1ab2eff8660": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f9475b046d642f8b5fe6631bac779b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "40d8e33a68f94b96857a0dffa8b572e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1aa36282e3b64575a82bd86c78ad22ac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0a6344ac08534c268ee87ad81e47cec4",
              "IPY_MODEL_0f54d9a8f1754c5a8673daca2881edfc"
            ]
          }
        },
        "1aa36282e3b64575a82bd86c78ad22ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0a6344ac08534c268ee87ad81e47cec4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_56a9626e01f846549352018707cce2dc",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 898823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_75c8cbf3b8c54278828ef602ff5b34c6"
          }
        },
        "0f54d9a8f1754c5a8673daca2881edfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e93109b5148c466a81250a48a0be53a9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:01&lt;00:00, 691kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7111ec402ca44e268f950b6d8de0cd08"
          }
        },
        "56a9626e01f846549352018707cce2dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "75c8cbf3b8c54278828ef602ff5b34c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e93109b5148c466a81250a48a0be53a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7111ec402ca44e268f950b6d8de0cd08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c4bfbfbf3c54de2804a0686e17df219": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7e70d2ed8b924482aa235acfb74fef40",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d44ccd7775d3499c853a2c454769b256",
              "IPY_MODEL_b6c06bba20644c63895ce15780a3b670"
            ]
          }
        },
        "7e70d2ed8b924482aa235acfb74fef40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d44ccd7775d3499c853a2c454769b256": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7d9125315d9446b19ae69f2d9e94a15e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8784d62a97034c718e01e12feaf18087"
          }
        },
        "b6c06bba20644c63895ce15780a3b670": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_959ac8978cd74fc698492aa1943ff62e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 1.39MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a64de82097414fe5b59f6d47e146c6ed"
          }
        },
        "7d9125315d9446b19ae69f2d9e94a15e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8784d62a97034c718e01e12feaf18087": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "959ac8978cd74fc698492aa1943ff62e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a64de82097414fe5b59f6d47e146c6ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3cadc34bdec14102856beb274b23cff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7589155debbb48c08bd5219dc53c09bb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d416ea53083b4f21a9fff2c8392759e5",
              "IPY_MODEL_900f190bf8454cc18d35990f4a6e929b"
            ]
          }
        },
        "7589155debbb48c08bd5219dc53c09bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d416ea53083b4f21a9fff2c8392759e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_50a866c0fabe4afb8a0bc9378433097b",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 230,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 230,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e3c31c1cf60a472a93ce912ab361ad49"
          }
        },
        "900f190bf8454cc18d35990f4a6e929b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_894332cabfe34e178330d27ccc0735e3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 230/230 [00:00&lt;00:00, 521B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d91b93d3a3dd45cd9a6bd82650dbe597"
          }
        },
        "50a866c0fabe4afb8a0bc9378433097b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e3c31c1cf60a472a93ce912ab361ad49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "894332cabfe34e178330d27ccc0735e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d91b93d3a3dd45cd9a6bd82650dbe597": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "291e617b16d3451c96c7e4e803c6b578": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d142e2eb3a6c4888b74a2973a76d257c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_94dab22c98104bf9a017bf97249264fc",
              "IPY_MODEL_fb4b129864dd47eb96b712534738e811"
            ]
          }
        },
        "d142e2eb3a6c4888b74a2973a76d257c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "94dab22c98104bf9a017bf97249264fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f557bc091c5f42708ca67c0afa24cbd5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 331070498,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 331070498,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e18cd4df56e0467b8a6df75bab9a2824"
          }
        },
        "fb4b129864dd47eb96b712534738e811": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fb2b6478a851441da1c5474765095333",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 331M/331M [00:07&lt;00:00, 47.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ee65825b1a9841c3aabd7e6d5dc42ca7"
          }
        },
        "f557bc091c5f42708ca67c0afa24cbd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e18cd4df56e0467b8a6df75bab9a2824": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb2b6478a851441da1c5474765095333": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ee65825b1a9841c3aabd7e6d5dc42ca7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a98ca567064f456986326e8c5c9d276d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8b48839571674aa1b34e7ec1360f3b3b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a5a1ec7889034fc789344da8addfcc0d",
              "IPY_MODEL_23f84b89d28d4f96a4737ac4fee38388"
            ]
          }
        },
        "8b48839571674aa1b34e7ec1360f3b3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a5a1ec7889034fc789344da8addfcc0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cf17cad027d24bf6a41c8c79f27e6c12",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cdff0a2201254c5984477a661daefcc9"
          }
        },
        "23f84b89d28d4f96a4737ac4fee38388": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95f5246006674300b775ca252314c3ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1000/1000 [11:42&lt;00:00,  1.42it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f2e29e6416e444e8afcdb7bf312efcc"
          }
        },
        "cf17cad027d24bf6a41c8c79f27e6c12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cdff0a2201254c5984477a661daefcc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95f5246006674300b775ca252314c3ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f2e29e6416e444e8afcdb7bf312efcc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/NER_data_augmentation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fECt2EcY2n2C",
        "colab_type": "text"
      },
      "source": [
        "# Data augmentation with transformer models for named entity recognition\n",
        "\n",
        "> Article: https://www.depends-on-the-definition.com/data-augmentation-with-transformers/ by [Tobias Sterbak](https://twitter.com/tobias_sterbak)\n",
        "\n",
        "> Colab Creator: [Manu Romero](https://twitter.com/mrm8488)\n",
        "\n",
        "![image](https://d33wubrfki0l68.cloudfront.net/cd99f03175460ab0a43c84b1ad7802adb4409295/3c45b/images/data-augmentation-with-transformers_files/data_augmentation.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P9MCiUXz9up9",
        "colab_type": "text"
      },
      "source": [
        "### Download the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PXSKamZT1dOd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        },
        "outputId": "5877f687-6cbe-4cc1-9918-76e657149fcb"
      },
      "source": [
        "!wget https://raw.githubusercontent.com/mrm8488/NER-English/master/ner_dataset.csv"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-08-24 10:06:50--  https://raw.githubusercontent.com/mrm8488/NER-English/master/ner_dataset.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 15208151 (15M) [text/plain]\n",
            "Saving to: ‘ner_dataset.csv’\n",
            "\n",
            "\rner_dataset.csv       0%[                    ]       0  --.-KB/s               \rner_dataset.csv      47%[========>           ]   6.83M  34.1MB/s               \rner_dataset.csv     100%[===================>]  14.50M  50.0MB/s    in 0.3s    \n",
            "\n",
            "2020-08-24 10:06:50 (50.0 MB/s) - ‘ner_dataset.csv’ saved [15208151/15208151]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_4JnCVCC90th",
        "colab_type": "text"
      },
      "source": [
        "### Install and import required libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u51PmPmg9xnr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e6qEbvim2P-K",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "10781f78-602f-4e51-b374-80d7f90c108b"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "from tqdm.notebook import tqdm\n",
        "\n",
        "torch.manual_seed(2020)\n",
        "\n",
        "print(torch.cuda.get_device_name(torch.cuda.current_device()))\n",
        "print(torch.cuda.is_available())\n",
        "print(torch.__version__)\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tesla P100-PCIE-16GB\n",
            "True\n",
            "1.6.0+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_RignTi9-Eh",
        "colab_type": "text"
      },
      "source": [
        "### Load the dataset and inspect it"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7PhZhR6L3qLP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "data = pd.read_csv(\"ner_dataset.csv\", encoding=\"latin1\")\n",
        "data = data.fillna(method=\"ffill\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zo2FFSyQ3tvA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 648
        },
        "outputId": "7a3159ac-ff55-4aad-d81d-b3333021d613"
      },
      "source": [
        "data.head(20)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>through</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>London</td>\n",
              "      <td>NNP</td>\n",
              "      <td>B-geo</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>to</td>\n",
              "      <td>TO</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>protest</td>\n",
              "      <td>VB</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>war</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>in</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Iraq</td>\n",
              "      <td>NNP</td>\n",
              "      <td>B-geo</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>and</td>\n",
              "      <td>CC</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>demand</td>\n",
              "      <td>VB</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>withdrawal</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>British</td>\n",
              "      <td>JJ</td>\n",
              "      <td>B-gpe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>troops</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     Sentence #           Word  POS    Tag\n",
              "0   Sentence: 1      Thousands  NNS      O\n",
              "1   Sentence: 1             of   IN      O\n",
              "2   Sentence: 1  demonstrators  NNS      O\n",
              "3   Sentence: 1           have  VBP      O\n",
              "4   Sentence: 1        marched  VBN      O\n",
              "5   Sentence: 1        through   IN      O\n",
              "6   Sentence: 1         London  NNP  B-geo\n",
              "7   Sentence: 1             to   TO      O\n",
              "8   Sentence: 1        protest   VB      O\n",
              "9   Sentence: 1            the   DT      O\n",
              "10  Sentence: 1            war   NN      O\n",
              "11  Sentence: 1             in   IN      O\n",
              "12  Sentence: 1           Iraq  NNP  B-geo\n",
              "13  Sentence: 1            and   CC      O\n",
              "14  Sentence: 1         demand   VB      O\n",
              "15  Sentence: 1            the   DT      O\n",
              "16  Sentence: 1     withdrawal   NN      O\n",
              "17  Sentence: 1             of   IN      O\n",
              "18  Sentence: 1        British   JJ  B-gpe\n",
              "19  Sentence: 1         troops  NNS      O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbbBELJY3uw6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 648
        },
        "outputId": "82808212-386e-44ef-9423-214b848ae015"
      },
      "source": [
        "data.tail(20)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1048555</th>\n",
              "      <td>Sentence: 47957</td>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048556</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>They</td>\n",
              "      <td>PRP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048557</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>say</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048558</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>not</td>\n",
              "      <td>RB</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048559</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>all</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048560</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048561</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048562</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>rockets</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048563</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>exploded</td>\n",
              "      <td>VBD</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048564</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>upon</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048565</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>impact</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048566</th>\n",
              "      <td>Sentence: 47958</td>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048567</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>Indian</td>\n",
              "      <td>JJ</td>\n",
              "      <td>B-gpe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048568</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>forces</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048569</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>said</td>\n",
              "      <td>VBD</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048570</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>they</td>\n",
              "      <td>PRP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048571</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>responded</td>\n",
              "      <td>VBD</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048572</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>to</td>\n",
              "      <td>TO</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048573</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048574</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>attack</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              Sentence #       Word  POS    Tag\n",
              "1048555  Sentence: 47957          .    .      O\n",
              "1048556  Sentence: 47958       They  PRP      O\n",
              "1048557  Sentence: 47958        say  VBP      O\n",
              "1048558  Sentence: 47958        not   RB      O\n",
              "1048559  Sentence: 47958        all   DT      O\n",
              "1048560  Sentence: 47958         of   IN      O\n",
              "1048561  Sentence: 47958        the   DT      O\n",
              "1048562  Sentence: 47958    rockets  NNS      O\n",
              "1048563  Sentence: 47958   exploded  VBD      O\n",
              "1048564  Sentence: 47958       upon   IN      O\n",
              "1048565  Sentence: 47958     impact   NN      O\n",
              "1048566  Sentence: 47958          .    .      O\n",
              "1048567  Sentence: 47959     Indian   JJ  B-gpe\n",
              "1048568  Sentence: 47959     forces  NNS      O\n",
              "1048569  Sentence: 47959       said  VBD      O\n",
              "1048570  Sentence: 47959       they  PRP      O\n",
              "1048571  Sentence: 47959  responded  VBD      O\n",
              "1048572  Sentence: 47959         to   TO      O\n",
              "1048573  Sentence: 47959        the   DT      O\n",
              "1048574  Sentence: 47959     attack   NN      O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "emm5iqEi36Ab",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 345
        },
        "outputId": "fccea964-bf45-4d56-93cc-0796f0f00387"
      },
      "source": [
        "data['Tag'].value_counts()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "O        887908\n",
              "B-geo     37644\n",
              "B-tim     20333\n",
              "B-org     20143\n",
              "I-per     17251\n",
              "B-per     16990\n",
              "I-org     16784\n",
              "B-gpe     15870\n",
              "I-geo      7414\n",
              "I-tim      6528\n",
              "B-art       402\n",
              "B-eve       308\n",
              "I-art       297\n",
              "I-eve       253\n",
              "B-nat       201\n",
              "I-gpe       198\n",
              "I-nat        51\n",
              "Name: Tag, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d1LfGha0-DgG",
        "colab_type": "text"
      },
      "source": [
        "#### Create a sentence getter"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GrkuNtJ44Bpi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SentenceGetter(object):\n",
        "    \n",
        "    def __init__(self, data):\n",
        "        self.n_sent = 1\n",
        "        self.data = data\n",
        "        self.empty = False\n",
        "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"Word\"].values.tolist(),\n",
        "                                                           s[\"POS\"].values.tolist(),\n",
        "                                                           s[\"Tag\"].values.tolist())]\n",
        "        self.grouped = self.data.groupby(\"Sentence #\").apply(agg_func)\n",
        "        self.sentences = [s for s in self.grouped]\n",
        "    \n",
        "    def get_next(self):\n",
        "        try:\n",
        "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
        "            self.n_sent += 1\n",
        "            return s\n",
        "        except:\n",
        "            return None\n"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v2L0z8Qo4LNk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "getter = SentenceGetter(data)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "66ZHQkNw4V6h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentences = getter.sentences"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UXndazqN4WjK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 454
        },
        "outputId": "95f2b684-3b1b-4be1-99e0-41d0d28daa2e"
      },
      "source": [
        "sentences[0]"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Thousands', 'NNS', 'O'),\n",
              " ('of', 'IN', 'O'),\n",
              " ('demonstrators', 'NNS', 'O'),\n",
              " ('have', 'VBP', 'O'),\n",
              " ('marched', 'VBN', 'O'),\n",
              " ('through', 'IN', 'O'),\n",
              " ('London', 'NNP', 'B-geo'),\n",
              " ('to', 'TO', 'O'),\n",
              " ('protest', 'VB', 'O'),\n",
              " ('the', 'DT', 'O'),\n",
              " ('war', 'NN', 'O'),\n",
              " ('in', 'IN', 'O'),\n",
              " ('Iraq', 'NNP', 'B-geo'),\n",
              " ('and', 'CC', 'O'),\n",
              " ('demand', 'VB', 'O'),\n",
              " ('the', 'DT', 'O'),\n",
              " ('withdrawal', 'NN', 'O'),\n",
              " ('of', 'IN', 'O'),\n",
              " ('British', 'JJ', 'B-gpe'),\n",
              " ('troops', 'NNS', 'O'),\n",
              " ('from', 'IN', 'O'),\n",
              " ('that', 'DT', 'O'),\n",
              " ('country', 'NN', 'O'),\n",
              " ('.', '.', 'O')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hacRCH0e4ZJc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tags = [\"[PAD]\"]\n",
        "tags.extend(list(set(data[\"Tag\"].values)))\n",
        "tag2idx = {t: i for i, t in enumerate(tags)}\n",
        "\n",
        "words = [\"[PAD]\", \"[UNK]\"]\n",
        "words.extend(list(set(data[\"Word\"].values)))\n",
        "word2idx = {t: i for i, t in enumerate(words)}"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q6fZZX8Z4vde",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_sentences, val_sentences, train_sentences = sentences[:15000], sentences[15000:20000], sentences[20000:]"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yo-QRUNt4zW4",
        "colab_type": "text"
      },
      "source": [
        "### Build a data augmentor with a transformer model\n",
        "This is based on HF/transformers ```fill-mask``` pipeline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BckskHcn445U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "from transformers import pipeline"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uAVXNvTG5ChS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class TransformerAugmenter():\n",
        "    \"\"\"\n",
        "    Use the pretrained masked language model to generate more\n",
        "    labeled samples from one labeled sentence.\n",
        "    \"\"\"\n",
        "    \n",
        "    def __init__(self):\n",
        "        self.num_sample_tokens = 5\n",
        "        self.fill_mask = pipeline(\n",
        "            \"fill-mask\",\n",
        "            topk=self.num_sample_tokens,\n",
        "            model=\"distilroberta-base\"\n",
        "        )\n",
        "    \n",
        "    def generate(self, sentence, num_replace_tokens=3):\n",
        "        \"\"\"Return a list of n augmented sentences.\"\"\"\n",
        "              \n",
        "        # run as often as tokens should be replaced\n",
        "        augmented_sentence = sentence.copy()\n",
        "        for i in range(num_replace_tokens):\n",
        "            # join the text\n",
        "            text = \" \".join([w[0] for w in augmented_sentence])\n",
        "            # pick a token\n",
        "            replace_token = random.choice(augmented_sentence)\n",
        "            # mask the picked token\n",
        "            masked_text = text.replace(\n",
        "                replace_token[0],\n",
        "                f\"{self.fill_mask.tokenizer.mask_token}\",\n",
        "                1            \n",
        "            )\n",
        "            # fill in the masked token with distilRoBERTa\n",
        "            res = self.fill_mask(masked_text)[random.choice(range(self.num_sample_tokens))]\n",
        "            # create output samples list\n",
        "            tmp_sentence, augmented_sentence = augmented_sentence.copy(), []\n",
        "            for w in tmp_sentence:\n",
        "                if w[0] == replace_token[0]:\n",
        "                    augmented_sentence.append((res[\"token_str\"].replace(\"Ġ\", \"\"), w[1], w[2]))\n",
        "                else:\n",
        "                    augmented_sentence.append(w)\n",
        "            text = \" \".join([w[0] for w in augmented_sentence])\n",
        "        return [sentence, augmented_sentence]\n"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6ZLhu_F6Llb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 361,
          "referenced_widgets": [
            "67eed08b809f49cdbde2a7adfa384826",
            "c0dbb696d1a3491fac930edd6c2edba1",
            "4e9cfe21abae410f9550e15cce5b12ae",
            "9e003f0e920749ba932db90a54efcad5",
            "3315bc7ca998472d97c7e321ce7e23c1",
            "4c5b63100a1741fdb2ac079c7042b583",
            "879ac1a5bc19487c9bafe1ab2eff8660",
            "2f9475b046d642f8b5fe6631bac779b3",
            "40d8e33a68f94b96857a0dffa8b572e4",
            "1aa36282e3b64575a82bd86c78ad22ac",
            "0a6344ac08534c268ee87ad81e47cec4",
            "0f54d9a8f1754c5a8673daca2881edfc",
            "56a9626e01f846549352018707cce2dc",
            "75c8cbf3b8c54278828ef602ff5b34c6",
            "e93109b5148c466a81250a48a0be53a9",
            "7111ec402ca44e268f950b6d8de0cd08",
            "8c4bfbfbf3c54de2804a0686e17df219",
            "7e70d2ed8b924482aa235acfb74fef40",
            "d44ccd7775d3499c853a2c454769b256",
            "b6c06bba20644c63895ce15780a3b670",
            "7d9125315d9446b19ae69f2d9e94a15e",
            "8784d62a97034c718e01e12feaf18087",
            "959ac8978cd74fc698492aa1943ff62e",
            "a64de82097414fe5b59f6d47e146c6ed",
            "3cadc34bdec14102856beb274b23cff0",
            "7589155debbb48c08bd5219dc53c09bb",
            "d416ea53083b4f21a9fff2c8392759e5",
            "900f190bf8454cc18d35990f4a6e929b",
            "50a866c0fabe4afb8a0bc9378433097b",
            "e3c31c1cf60a472a93ce912ab361ad49",
            "894332cabfe34e178330d27ccc0735e3",
            "d91b93d3a3dd45cd9a6bd82650dbe597",
            "291e617b16d3451c96c7e4e803c6b578",
            "d142e2eb3a6c4888b74a2973a76d257c",
            "94dab22c98104bf9a017bf97249264fc",
            "fb4b129864dd47eb96b712534738e811",
            "f557bc091c5f42708ca67c0afa24cbd5",
            "e18cd4df56e0467b8a6df75bab9a2824",
            "fb2b6478a851441da1c5474765095333",
            "ee65825b1a9841c3aabd7e6d5dc42ca7"
          ]
        },
        "outputId": "6f37feee-12af-441a-94f7-fee752f30fec"
      },
      "source": [
        "augmenter = TransformerAugmenter()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "67eed08b809f49cdbde2a7adfa384826",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=480.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "40d8e33a68f94b96857a0dffa8b572e4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c4bfbfbf3c54de2804a0686e17df219",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3cadc34bdec14102856beb274b23cff0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=230.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/modeling_auto.py:798: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "291e617b16d3451c96c7e4e803c6b578",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=331070498.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of RobertaForMaskedLM were not initialized from the model checkpoint at distilroberta-base and are newly initialized: ['lm_head.decoder.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJTB1_qp6cPK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "outputId": "9ad7556a-8377-46ed-c620-d2a765130015"
      },
      "source": [
        "train_sentences[0]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('In', 'IN', 'O'),\n",
              " ('other', 'JJ', 'O'),\n",
              " ('violence', 'NN', 'O'),\n",
              " (',', ',', 'O'),\n",
              " ('U.S.', 'NNP', 'B-gpe'),\n",
              " ('officials', 'NNS', 'O'),\n",
              " ('said', 'VBD', 'O'),\n",
              " ('one', 'CD', 'O'),\n",
              " ('American', 'JJ', 'B-gpe'),\n",
              " ('soldier', 'NN', 'O'),\n",
              " ('was', 'VBD', 'O'),\n",
              " ('killed', 'VBN', 'O'),\n",
              " ('while', 'IN', 'O'),\n",
              " ('on', 'IN', 'O'),\n",
              " ('patrol', 'NN', 'O'),\n",
              " ('in', 'IN', 'O'),\n",
              " ('Baghdad', 'NNP', 'B-geo'),\n",
              " ('Sunday', 'NNP', 'B-tim'),\n",
              " ('.', '.', 'O')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vmCI5vi6McE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 799
        },
        "outputId": "7802c993-a3bb-4704-ab67-32755e090249"
      },
      "source": [
        "augmented_sentences = augmenter.generate(train_sentences[0], num_replace_tokens=7); augmented_sentences"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/pipelines.py:882: UserWarning: This overload of nonzero is deprecated:\n",
            "\tnonzero()\n",
            "Consider using one of the following signatures instead:\n",
            "\tnonzero(*, bool as_tuple) (Triggered internally at  /pytorch/torch/csrc/utils/python_arg_parser.cpp:766.)\n",
            "  masked_index = (input_ids == self.tokenizer.mask_token_id).nonzero()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[('In', 'IN', 'O'),\n",
              "  ('other', 'JJ', 'O'),\n",
              "  ('violence', 'NN', 'O'),\n",
              "  (',', ',', 'O'),\n",
              "  ('U.S.', 'NNP', 'B-gpe'),\n",
              "  ('officials', 'NNS', 'O'),\n",
              "  ('said', 'VBD', 'O'),\n",
              "  ('one', 'CD', 'O'),\n",
              "  ('American', 'JJ', 'B-gpe'),\n",
              "  ('soldier', 'NN', 'O'),\n",
              "  ('was', 'VBD', 'O'),\n",
              "  ('killed', 'VBN', 'O'),\n",
              "  ('while', 'IN', 'O'),\n",
              "  ('on', 'IN', 'O'),\n",
              "  ('patrol', 'NN', 'O'),\n",
              "  ('in', 'IN', 'O'),\n",
              "  ('Baghdad', 'NNP', 'B-geo'),\n",
              "  ('Sunday', 'NNP', 'B-tim'),\n",
              "  ('.', '.', 'O')],\n",
              " [('among', 'IN', 'O'),\n",
              "  ('other', 'JJ', 'O'),\n",
              "  ('violence', 'NN', 'O'),\n",
              "  (',', ',', 'O'),\n",
              "  ('U.S.', 'NNP', 'B-gpe'),\n",
              "  ('officials', 'NNS', 'O'),\n",
              "  ('said', 'VBD', 'O'),\n",
              "  ('one', 'CD', 'O'),\n",
              "  ('American', 'JJ', 'B-gpe'),\n",
              "  ('soldier', 'NN', 'O'),\n",
              "  ('was', 'VBD', 'O'),\n",
              "  ('killed', 'VBN', 'O'),\n",
              "  ('when', 'IN', 'O'),\n",
              "  ('on', 'IN', 'O'),\n",
              "  ('patrol', 'NN', 'O'),\n",
              "  ('in', 'IN', 'O'),\n",
              "  ('Baghdad', 'NNP', 'B-geo'),\n",
              "  ('Sunday', 'NNP', 'B-tim'),\n",
              "  ('.-', '.', 'O')]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "au41BJiO8KbM",
        "colab_type": "text"
      },
      "source": [
        "### Generate an augmented dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POuea1aM6hWk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c32e263d-de0d-4e4c-fcd6-f48145564e97"
      },
      "source": [
        "len(train_sentences)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "27959"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XTf5Alks8Omh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "a98ca567064f456986326e8c5c9d276d",
            "8b48839571674aa1b34e7ec1360f3b3b",
            "a5a1ec7889034fc789344da8addfcc0d",
            "23f84b89d28d4f96a4737ac4fee38388",
            "cf17cad027d24bf6a41c8c79f27e6c12",
            "cdff0a2201254c5984477a661daefcc9",
            "95f5246006674300b775ca252314c3ee",
            "2f2e29e6416e444e8afcdb7bf312efcc"
          ]
        },
        "outputId": "db13e220-0d97-4e16-8b89-761e6a20a4ae"
      },
      "source": [
        "# only use a thousand senteces with augmentation (it can take about 10 mins)\n",
        "n_sentences = 1000\n",
        "\n",
        "augmented_sentences = []\n",
        "for sentence in tqdm(train_sentences[:n_sentences]):\n",
        "    augmented_sentences.extend(augmenter.generate(sentence, num_replace_tokens=7))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a98ca567064f456986326e8c5c9d276d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V8d9pFY08aDk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7d99a038-9fb1-4ca5-947e-7d058e184ec3"
      },
      "source": [
        "len(augmented_sentences)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    }
  ]
}