{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4", "collapsed_sections": [ "-8kEDRvShcU5", "viKGCCh8izww", "aRp4O7D295d_", "7LzSbTqW9_BR", "8bwa6Rts-02-" ] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "abd038facca842aaad89f90c94160d30": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_3a3f1bbd157f4bddb1310032cd323338", "IPY_MODEL_2abd143f80804cbfa7150d60a9497d4f", "IPY_MODEL_a07e5dfec1e343dbad40b1b9645b8e06" ], "layout": "IPY_MODEL_4bd337d8f4664c7687dbb573bd1e09a0" } }, "3a3f1bbd157f4bddb1310032cd323338": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_390edf5d989f431096797bcf135d4c9f", "placeholder": "​", "style": "IPY_MODEL_6d07b45949224afabac5e0b436a44684", "value": "tokenizer_config.json: 100%" } }, "2abd143f80804cbfa7150d60a9497d4f": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0356864f689a4235b0b4ad8029246642", "max": 48, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_e21f68537bfc44bd96c2f06dfff613b8", "value": 48 } }, "a07e5dfec1e343dbad40b1b9645b8e06": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c708e30153e64dfab92b4f41f68baa3d", "placeholder": "​", "style": "IPY_MODEL_1ef36450d205460ab7d1db3cbcd52d11", "value": " 48.0/48.0 [00:00<00:00, 3.57kB/s]" } }, "4bd337d8f4664c7687dbb573bd1e09a0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "390edf5d989f431096797bcf135d4c9f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6d07b45949224afabac5e0b436a44684": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "0356864f689a4235b0b4ad8029246642": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e21f68537bfc44bd96c2f06dfff613b8": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "c708e30153e64dfab92b4f41f68baa3d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1ef36450d205460ab7d1db3cbcd52d11": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "35d0ca607f644e739d354a478f456d02": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_d74b3332e1c346dfad5a1d718b5d47f7", "IPY_MODEL_26930dd4f63a4f92830322464a7faf48", "IPY_MODEL_590e975992974e5f99ec4fe9c09cf941" ], "layout": "IPY_MODEL_f43a24c59f344cb588bb602d49146a8a" } }, "d74b3332e1c346dfad5a1d718b5d47f7": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_134524c137904cf29f18acc3e2cf80bd", "placeholder": "​", "style": "IPY_MODEL_1bc55b7cd69f4793a5171918a72e8c16", "value": "vocab.txt: 100%" } }, "26930dd4f63a4f92830322464a7faf48": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_35e13b8b82ba4693b3b5cd478172dbbc", "max": 231508, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_dbcb362e6a864851a91fda8aabf18f5b", "value": 231508 } }, "590e975992974e5f99ec4fe9c09cf941": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bb80001918fc4453a2fd98e4b9698fab", "placeholder": "​", "style": "IPY_MODEL_d03d1fcafdc443f7907cc3958163ec8d", "value": " 232k/232k [00:00<00:00, 6.10MB/s]" } }, "f43a24c59f344cb588bb602d49146a8a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "134524c137904cf29f18acc3e2cf80bd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "1bc55b7cd69f4793a5171918a72e8c16": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "35e13b8b82ba4693b3b5cd478172dbbc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "dbcb362e6a864851a91fda8aabf18f5b": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "bb80001918fc4453a2fd98e4b9698fab": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d03d1fcafdc443f7907cc3958163ec8d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "2af5c6f21f7f4ef4968a2a4eb7485c2d": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_58e8f882b9194074bab3fb2346d2e8b9", "IPY_MODEL_1f3559f34eae45c7b2a33944bf82ac73", "IPY_MODEL_deae7cf343994d47abe85d3cc84e152a" ], "layout": "IPY_MODEL_a817b6db322a4350a166389766dbf0bd" } }, "58e8f882b9194074bab3fb2346d2e8b9": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_15ef467f76ac4dee94e5fdf8458a6603", "placeholder": "​", "style": "IPY_MODEL_be052e0ca60146398b3a5e425a3f7464", "value": "tokenizer.json: 100%" } }, "1f3559f34eae45c7b2a33944bf82ac73": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2c84f9b0282f4c2991ca0e1994c0565e", "max": 466062, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_5eaf012fc77e46acb41f10e219192aed", "value": 466062 } }, "deae7cf343994d47abe85d3cc84e152a": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_dbeedea5e0cd4f7780ecec59d7e70560", "placeholder": "​", "style": "IPY_MODEL_bc607e51150f4d84b0a8705d812f3b66", "value": " 466k/466k [00:00<00:00, 20.6MB/s]" } }, "a817b6db322a4350a166389766dbf0bd": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "15ef467f76ac4dee94e5fdf8458a6603": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "be052e0ca60146398b3a5e425a3f7464": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "2c84f9b0282f4c2991ca0e1994c0565e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5eaf012fc77e46acb41f10e219192aed": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "dbeedea5e0cd4f7780ecec59d7e70560": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "bc607e51150f4d84b0a8705d812f3b66": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "83f93de49175457f8eafe81dcadc797d": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_b63bf312c2954982b25b30392283bfcc", "IPY_MODEL_cc14e297ca5140cdb7e5f2929be7bbbf", "IPY_MODEL_a8745dbd05be474ea97d2e0b6ba72f8e" ], "layout": "IPY_MODEL_05b2ada75dbf43e88babcce42a836420" } }, "b63bf312c2954982b25b30392283bfcc": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_afd62463d5ef4494a57e2e61d4495880", "placeholder": "​", "style": "IPY_MODEL_902c236ca2d04191a24c4c6a67033369", "value": "config.json: 100%" } }, "cc14e297ca5140cdb7e5f2929be7bbbf": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e88a3142031840f6b6a00109f346c564", "max": 570, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_5c9340e6ac554dcda760929a7e30d3aa", "value": 570 } }, "a8745dbd05be474ea97d2e0b6ba72f8e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3715ebaf119e47ab895773005ef23434", "placeholder": "​", "style": "IPY_MODEL_87ea861b941f4eb68e16b60dbc77d98a", "value": " 570/570 [00:00<00:00, 32.1kB/s]" } }, "05b2ada75dbf43e88babcce42a836420": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "afd62463d5ef4494a57e2e61d4495880": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "902c236ca2d04191a24c4c6a67033369": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e88a3142031840f6b6a00109f346c564": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5c9340e6ac554dcda760929a7e30d3aa": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "3715ebaf119e47ab895773005ef23434": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "87ea861b941f4eb68e16b60dbc77d98a": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "EKOTlwcmxmej" }, "source": [ "# Sentence Classification with BERT\n", "\n", "\n", "\n" ] }, { "cell_type": "markdown", "source": [ "In this assignment, we're going to investigate Bert performance on a sentence classification task" ], "metadata": { "id": "fm89Jxo1F7Me" } }, { "cell_type": "markdown", "metadata": { "id": "RX_ZDhicpHkV" }, "source": [ "# 1. Setup" ] }, { "cell_type": "markdown", "metadata": { "id": "nSU7yERLP_66" }, "source": [ "## 1.1. Using Colab GPU for Training\n" ] }, { "cell_type": "markdown", "metadata": { "id": "cqG7FzRVFEIv" }, "source": [ "Specify GPU for training." ] }, { "cell_type": "code", "metadata": { "id": "oYsV4H8fCpZ-", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "407948b7-aa95-44f1-fb0a-7853621f220e" }, "source": [ "import torch\n", "\n", "# If there's a GPU available...\n", "if torch.cuda.is_available():\n", "\n", " # Tell PyTorch to use the GPU.\n", " device = torch.device(\"cuda\")\n", "\n", " print('There are %d GPU(s) available.' % torch.cuda.device_count())\n", "\n", " print('We will use the GPU:', torch.cuda.get_device_name(0))\n", "\n", "# If not...\n", "else:\n", " print('No GPU available, using the CPU instead.')\n", " device = torch.device(\"cpu\")" ], "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "There are 1 GPU(s) available.\n", "We will use the GPU: Tesla T4\n" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "2ElsnSNUridI" }, "source": [ "## 1.2. Installing the Hugging Face Library\n" ] }, { "cell_type": "markdown", "metadata": { "id": "G_N2UDLevYWn" }, "source": [ "\n", "\n", "---\n", "To use BERT in PyTorch, install [transformers](https://github.com/huggingface/transformers) package from Hugging Face. It offers a PyTorch interface for BERT and other pretrained language models like GPT and GPT-2.\n", "\n", "Hugging Face provides is widely used for BERT-related tasks. In our assignment, we'll focus on using BertForSequenceClassification.\n", "\n", "\n", "---\n" ] }, { "cell_type": "code", "metadata": { "id": "0NmMdkZO8R6q", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "8c22c32e-8d53-4fb1-f0c5-f933c64dd62e" }, "source": [ "!pip install transformers" ], "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.38.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n", "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n", "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.2)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.6.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.10.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "guw6ZNtaswKc" }, "source": [ "# 2. Loading Dataset\n" ] }, { "cell_type": "markdown", "metadata": { "id": "4SMZ5T5Imhlx" }, "source": [ "We can use the helper function given in the previous part of asignment 3 to extract the sentences and labels of our training set." ] }, { "cell_type": "code", "source": [ "from google.colab import drive\n", "drive.mount('/content/drive')\n" ], "metadata": { "id": "xGpHJMh_Bgx-", "outputId": "f5ea7a9e-f796-408c-f34e-1adc3bbb113e", "colab": { "base_uri": "https://localhost:8080/" } }, "execution_count": 16, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n" ] } ] }, { "cell_type": "code", "source": [ "# cd /content/drive/MyDrive/BERT\\ text\\ classification\\ with\\ pytorch/\n", "import os\n", "os.chdir(\"/content/drive/MyDrive/ColabNotebooks/\")\n", "!pwd\n", "!ls" ], "metadata": { "id": "kvmWqwJbBzOD", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "0ed68aa6-b847-4642-9397-ad0c0e7e5ee5" }, "execution_count": 29, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "/content/drive/MyDrive/ColabNotebooks\n", " 20news_subsampled\t\t\t chapter3.ipynb\n", "'Assignment1_proj (1).ipynb'\t\t Chapter4.ipynb\n", " Assignment1_proj_2024.ipynb\t\t 'Copy of 05.09-Principal-Component-Analysis.ipynb'\n", " Assignment1_proj.ipynb\t\t\t 'Copy of assignment2_proj_solution_oliver.ipynb'\n", " Assignment1_proj_sol_2024.ipynb\t Copy_of_Sentence_Classification_with_BERT_4.ipynb\n", " Assignment1_proj_sol.ipynb\t\t 'Copy of Tensor2Tensor Intro'\n", "'Assignment 1-Solution.ipynb'\t\t 'Copy of Tensor2Tensor Intro by kiarash'\n", "'Assignment 1-Solution-keras.ipynb'\t 'create_database carla271_1.ipynb'\n", " assignment2_proj.ipynb\t\t\t Dataset\n", " assignment2_proj_solution.ipynb\t 'English Literature.txt'\n", " assignment2_proj_solution_kiarash.ipynb PCADemo.ipynb\n", " assignment2_proj_solution_oliver.ipynb Principal-Component-Analysis.ipynb\n", " assignment3_proj.ipynb\t\t\t timeslice2.ipynb\n", " assignment3_proj_solution.ipynb\t train.csv\n", "'Assignment Two-released.ipynb'\t\t Untitled0.ipynb\n", "'Assignment Two Solution.ipynb'\t\t validation.csv\n" ] } ] }, { "cell_type": "code", "source": [ "\"\"\"This code is used to read all news and their labels\"\"\"\n", "import os\n", "import glob\n", "\n", "def to_categories(name, cat=[\"politics\",\"rec\",\"comp\",\"religion\"]):\n", " for i in range(len(cat)):\n", " if str.find(name,cat[i])>-1:\n", " return(i)\n", " print(\"Unexpected folder: \" + name) # print the folder name which does not include expected categories\n", " return(\"wth\")\n", "\n", "def data_loader(images_dir):\n", " categories = os.listdir(data_path)\n", " news = [] # news content\n", " groups = [] # category which it belong to\n", "\n", " for cat in categories:\n", " print(\"Category:\"+cat)\n", " for the_new_path in glob.glob(data_path + '/' + cat + '/*'):\n", " news.append(open(the_new_path,encoding = \"ISO-8859-1\", mode ='r').read())\n", " groups.append(cat)\n", "\n", " return news, list(map(to_categories, groups))\n", "\n", "\n", "\n", "data_path = \"20news_subsampled\"\n", "news, groups = data_loader(data_path)" ], "metadata": { "id": "SOKOFqGLB6uh", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "4b02ac50-ce79-4616-f074-02aab7aaf4f5" }, "execution_count": 30, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Category:rec.autos\n", "Category:talk.politics.misc\n", "Category:rec.motorcycles\n", "Category:comp.windows.x\n", "Category:comp.graphics\n", "Category:comp.sys.ibm.pc.hardware\n", "Category:talk.politics.guns\n", "Category:comp.sys.mac.hardware\n", "Category:rec.sport.baseball\n", "Category:comp.os.ms-windows.misc\n", "Category:rec.sport.hockey\n", "Category:soc.religion.christian\n", "Category:talk.politics.mideast\n", "Category:talk.religion.misc\n" ] } ] }, { "cell_type": "code", "source": [ "import numpy as np\n", "sentences=np.array(news)\n", "labels=np.array(groups)" ], "metadata": { "id": "pMfpa4HfEhbf" }, "execution_count": 31, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "ex5O1eV-Pfct" }, "source": [ "# 3. Tokenization & Input Formatting\n", "\n", "In this section, we'll transform our dataset into the format that BERT can be trained on. You should complete step 3.2." ] }, { "cell_type": "markdown", "metadata": { "id": "-8kEDRvShcU5" }, "source": [ "## 3.1. BERT Tokenizer" ] }, { "cell_type": "markdown", "metadata": { "id": "bWOPOyWghJp2" }, "source": [ "In order to use BERT for text processing, we need to tokenize the text by splitting it into tokens and mapping these tokens to their corresponding indices in the tokenizer vocabulary.\n", "\n", "To achieve this, we will utilize the tokenizer provided with BERT. The code snippet in the following cell will download the tokenizer for us. In this case, we will be using the \"uncased\" version of the tokenizer." ] }, { "cell_type": "code", "metadata": { "id": "Z474sSC6oe7A", "colab": { "base_uri": "https://localhost:8080/", "height": 288, "referenced_widgets": [ "abd038facca842aaad89f90c94160d30", "3a3f1bbd157f4bddb1310032cd323338", "2abd143f80804cbfa7150d60a9497d4f", "a07e5dfec1e343dbad40b1b9645b8e06", "4bd337d8f4664c7687dbb573bd1e09a0", "390edf5d989f431096797bcf135d4c9f", "6d07b45949224afabac5e0b436a44684", "0356864f689a4235b0b4ad8029246642", "e21f68537bfc44bd96c2f06dfff613b8", "c708e30153e64dfab92b4f41f68baa3d", "1ef36450d205460ab7d1db3cbcd52d11", "35d0ca607f644e739d354a478f456d02", "d74b3332e1c346dfad5a1d718b5d47f7", "26930dd4f63a4f92830322464a7faf48", "590e975992974e5f99ec4fe9c09cf941", "f43a24c59f344cb588bb602d49146a8a", "134524c137904cf29f18acc3e2cf80bd", "1bc55b7cd69f4793a5171918a72e8c16", "35e13b8b82ba4693b3b5cd478172dbbc", "dbcb362e6a864851a91fda8aabf18f5b", "bb80001918fc4453a2fd98e4b9698fab", "d03d1fcafdc443f7907cc3958163ec8d", "2af5c6f21f7f4ef4968a2a4eb7485c2d", "58e8f882b9194074bab3fb2346d2e8b9", "1f3559f34eae45c7b2a33944bf82ac73", "deae7cf343994d47abe85d3cc84e152a", "a817b6db322a4350a166389766dbf0bd", "15ef467f76ac4dee94e5fdf8458a6603", "be052e0ca60146398b3a5e425a3f7464", "2c84f9b0282f4c2991ca0e1994c0565e", "5eaf012fc77e46acb41f10e219192aed", "dbeedea5e0cd4f7780ecec59d7e70560", "bc607e51150f4d84b0a8705d812f3b66", "83f93de49175457f8eafe81dcadc797d", "b63bf312c2954982b25b30392283bfcc", "cc14e297ca5140cdb7e5f2929be7bbbf", "a8745dbd05be474ea97d2e0b6ba72f8e", "05b2ada75dbf43e88babcce42a836420", "afd62463d5ef4494a57e2e61d4495880", "902c236ca2d04191a24c4c6a67033369", "e88a3142031840f6b6a00109f346c564", "5c9340e6ac554dcda760929a7e30d3aa", "3715ebaf119e47ab895773005ef23434", "87ea861b941f4eb68e16b60dbc77d98a" ] }, "outputId": "e9baf600-8a67-4570-f47f-52bff1dc377e" }, "source": [ "from transformers import BertTokenizer\n", "\n", "# Load the BERT tokenizer.\n", "print('Loading BERT tokenizer...')\n", "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)" ], "execution_count": 32, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Loading BERT tokenizer...\n" ] }, { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n", "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", "You will be able to reuse this secret in all of your notebooks.\n", "Please note that authentication is recommended but still optional to access public models or datasets.\n", " warnings.warn(\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "tokenizer_config.json: 0%| | 0.00/48.0 [00:00\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# Use 90% for training and 10% for validation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels,\n\u001b[0m\u001b[1;32m 7\u001b[0m random_state=2018, test_size=0.1)\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# Do the same for the masks.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_split.py\u001b[0m in \u001b[0;36mtrain_test_split\u001b[0;34m(test_size, train_size, random_state, shuffle, stratify, *arrays)\u001b[0m\n\u001b[1;32m 2560\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2561\u001b[0m \u001b[0mn_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_num_samples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2562\u001b[0;31m n_train, n_test = _validate_shuffle_split(\n\u001b[0m\u001b[1;32m 2563\u001b[0m \u001b[0mn_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefault_test_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.25\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2564\u001b[0m )\n", "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/sklearn/model_selection/_split.py\u001b[0m in \u001b[0;36m_validate_shuffle_split\u001b[0;34m(n_samples, test_size, train_size, default_test_size)\u001b[0m\n\u001b[1;32m 2234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2235\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mn_train\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2236\u001b[0;31m raise ValueError(\n\u001b[0m\u001b[1;32m 2237\u001b[0m \u001b[0;34m\"With n_samples={}, test_size={} and train_size={}, the \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2238\u001b[0m \u001b[0;34m\"resulting train set will be empty. Adjust any of the \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mValueError\u001b[0m: With n_samples=0, test_size=0.1 and train_size=None, the resulting train set will be empty. Adjust any of the aforementioned parameters." ] } ] }, { "cell_type": "markdown", "metadata": { "id": "7LzSbTqW9_BR" }, "source": [ "## 3.4. Converting to PyTorch Data Types" ] }, { "cell_type": "markdown", "metadata": { "id": "6p1uXczp-Je4" }, "source": [ "Our model expects PyTorch tensors rather than numpy.ndarrays, so convert all of our dataset variables." ] }, { "cell_type": "code", "metadata": { "id": "jw5K2A5Ko1RF", "colab": { "base_uri": "https://localhost:8080/", "height": 233 }, "outputId": "b10adc7e-a4a9-453b-e2d3-6118b7992bc7" }, "source": [ "# Convert all inputs and labels into torch tensors, the required datatype\n", "# for our model.\n", "# You can also use the library torch.from_numpy\n", "\n", "train_inputs = torch.tensor(train_inputs)\n", "validation_inputs = torch.tensor(validation_inputs)\n", "\n", "train_labels = torch.tensor(train_labels)\n", "validation_labels = torch.tensor(validation_labels)\n", "\n", "train_masks = torch.tensor(train_masks)\n", "validation_masks = torch.tensor(validation_masks)" ], "execution_count": 41, "outputs": [ { "output_type": "error", "ename": "NameError", "evalue": "name 'train_inputs' is not defined", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;31m# for our model.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;31m# You can also use the library torch.from_numpy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mtrain_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0mvalidation_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mNameError\u001b[0m: name 'train_inputs' is not defined" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "dD9i6Z2pG-sN" }, "source": [ "We'll also create an iterator for our dataset using the torch DataLoader class." ] }, { "cell_type": "code", "metadata": { "id": "GEgLpFVlo1Z-" }, "source": [ "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n", "\n", "# The DataLoader needs to know our batch size for training, so we specify it\n", "# here.\n", "# For fine-tuning BERT on a specific task, the authors recommend a batch size of\n", "# 16 or 32.\n", "\n", "batch_size = 32\n", "\n", "# Create the DataLoader for our training set.\n", "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n", "train_sampler = RandomSampler(train_data)\n", "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n", "\n", "# Create the DataLoader for our validation set.\n", "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n", "validation_sampler = SequentialSampler(validation_data)\n", "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "8bwa6Rts-02-" }, "source": [ "# 4. Train Our Classification Model" ] }, { "cell_type": "markdown", "metadata": { "id": "3xYQ3iLO08SX" }, "source": [ "Now that our input data is properly formatted, it's time to fine tune the BERT model." ] }, { "cell_type": "markdown", "metadata": { "id": "D6TKgyUzPIQc" }, "source": [ "## 4.1. BertForSequenceClassification" ] }, { "cell_type": "markdown", "metadata": { "id": "BXYitPoE-cjH" }, "source": [ "\n", "\n", "We'll be using [BertForSequenceClassification](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#bertforsequenceclassification). This is the normal BERT model with an added single linear layer on top for classification that we will use as a sentence classifier. As we feed input data, the entire pre-trained BERT model and the additional untrained classification layer is trained on our specific task.\n" ] }, { "cell_type": "markdown", "metadata": { "id": "WnQW9E-bBCRt" }, "source": [ "Now, we will proceed to load BERT. There are several pre-trained BERT models to choose from. By selecting \"bert-base-uncased,\" we opt for the smaller version of BERT that operates with only lowercase text\n" ] }, { "cell_type": "code", "metadata": { "id": "gFsCTp_mporB", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "6797b48d-5dab-4c43-be68-5be196cf6f5a" }, "source": [ "from transformers import BertForSequenceClassification, AdamW, BertConfig\n", "\n", "# Load BertForSequenceClassification, the pretrained BERT model with a single\n", "# linear classification layer on top.\n", "model = BertForSequenceClassification.from_pretrained(\n", " \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n", " num_labels = 4, # The number of output labels\n", " output_attentions = False, # Whether the model returns attentions weights.\n", " output_hidden_states = False, # Whether the model returns all hidden-states.\n", ")\n", "\n", "# Tell pytorch to run this model on the GPU.\n", "model.cuda()" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "BertForSequenceClassification(\n", " (bert): BertModel(\n", " (embeddings): BertEmbeddings(\n", " (word_embeddings): Embedding(30522, 768, padding_idx=0)\n", " (position_embeddings): Embedding(512, 768)\n", " (token_type_embeddings): Embedding(2, 768)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (encoder): BertEncoder(\n", " (layer): ModuleList(\n", " (0-11): 12 x BertLayer(\n", " (attention): BertAttention(\n", " (self): BertSelfAttention(\n", " (query): Linear(in_features=768, out_features=768, bias=True)\n", " (key): Linear(in_features=768, out_features=768, bias=True)\n", " (value): Linear(in_features=768, out_features=768, bias=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (output): BertSelfOutput(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " (intermediate): BertIntermediate(\n", " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", " (intermediate_act_fn): GELUActivation()\n", " )\n", " (output): BertOutput(\n", " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (pooler): BertPooler(\n", " (dense): Linear(in_features=768, out_features=768, bias=True)\n", " (activation): Tanh()\n", " )\n", " )\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (classifier): Linear(in_features=768, out_features=4, bias=True)\n", ")" ] }, "metadata": {}, "execution_count": 28 } ] }, { "cell_type": "markdown", "metadata": { "id": "qRWT-D4U_Pvx" }, "source": [ "## 4.2. Optimizer & Learning Rate Scheduler" ] }, { "cell_type": "markdown", "metadata": { "id": "8o-VEBobKwHk" }, "source": [ "Now that we have our model loaded we need to grab the training hyperparameters from within the stored model.\n", "\n", "For the purposes of fine-tuning, the authors recommend choosing from the following values:\n", "- Batch size: 16, 32 (We chose 32 when creating our DataLoaders).\n", "- Learning rate (Adam): 5e-5, 3e-5, 2e-5 (We'll use 2e-5).\n", "- Number of epochs: 2, 3, 4 (We'll use 4).\n", "\n", "The epsilon parameter `eps = 1e-8` is \"a very small number to prevent any division by zero in the implementation\" (from [here](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n" ] }, { "cell_type": "code", "metadata": { "id": "GLs72DuMODJO", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "d7c64bbf-c674-4ee2-f149-ab9258fe1f2e" }, "source": [ "optimizer = AdamW(model.parameters(),\n", " lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n", " eps = 1e-8 # args.adam_epsilon - default is 1e-8.\n", " )\n" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:429: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] } ] }, { "cell_type": "code", "metadata": { "id": "-p0upAhhRiIx" }, "source": [ "from transformers import get_linear_schedule_with_warmup\n", "\n", "# Number of training epochs (authors recommend between 2 and 4)\n", "epochs = 4\n", "\n", "# Total number of training steps is number of batches * number of epochs.\n", "total_steps = len(train_dataloader) * epochs\n", "\n", "# Create the learning rate scheduler.\n", "scheduler = get_linear_schedule_with_warmup(optimizer,\n", " num_warmup_steps = 0, # Default value in run_glue.py\n", " num_training_steps = total_steps)" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "RqfmWwUR_Sox" }, "source": [ "## 4.3. Training Loop" ] }, { "cell_type": "markdown", "metadata": { "id": "_QXZhFb4LnV5" }, "source": [ "Below is our training loop. For each pass in our loop we have a trianing phase and a validation phase. At each pass we need to:\n", "\n", "Training loop:\n", "- Unpack our data inputs and labels\n", "- Load data onto the GPU for acceleration\n", "- Clear out the gradients calculated in the previous pass.\n", " - In pytorch the gradients accumulate by default (useful for things like RNNs) unless you explicitly clear them out.\n", "- Forward pass (feed input data through the network)\n", "- Backward pass (backpropagation)\n", "- Tell the network to update parameters with optimizer.step()\n", "- Track variables for monitoring progress\n", "\n", "Evalution loop:\n", "- Unpack our data inputs and labels\n", "- Load data onto the GPU for acceleration\n", "- Forward pass (feed input data through the network)\n", "- Compute loss on our validation data and track variables for monitoring progress\n", "\n" ] }, { "cell_type": "markdown", "metadata": { "id": "pE5B99H5H2-W" }, "source": [ "Define a helper function for calculating accuracy." ] }, { "cell_type": "code", "metadata": { "id": "9cQNvaZ9bnyy" }, "source": [ "import numpy as np\n", "\n", "# Function to calculate the accuracy of our predictions vs labels\n", "def flat_accuracy(preds, labels):\n", " pred_flat = np.argmax(preds, axis=1).flatten()\n", " labels_flat = labels.flatten()\n", " return np.sum(pred_flat == labels_flat) / len(labels_flat)" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "KNhRtWPXH9C3" }, "source": [ "Helper function for formatting elapsed times.\n" ] }, { "cell_type": "code", "metadata": { "id": "gpt6tR83keZD" }, "source": [ "import time\n", "import datetime\n", "\n", "def format_time(elapsed):\n", " '''\n", " Takes a time in seconds and returns a string hh:mm:ss\n", " '''\n", " # Round to the nearest second.\n", " elapsed_rounded = int(round((elapsed)))\n", "\n", " # Format as hh:mm:ss\n", " return str(datetime.timedelta(seconds=elapsed_rounded))\n" ], "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "cfNIhN19te3N" }, "source": [ "We're ready to kick off the training!" ] }, { "cell_type": "code", "metadata": { "id": "6J-FYdx6nFE_", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "24d40955-5e81-41c5-fd02-14515fb0bed3" }, "source": [ "import random\n", "\n", "# Set the seed value all over the place to make this reproducible.\n", "seed_val = 42\n", "\n", "random.seed(seed_val)\n", "np.random.seed(seed_val)\n", "torch.manual_seed(seed_val)\n", "torch.cuda.manual_seed_all(seed_val)\n", "\n", "# Store the average loss after each epoch so we can plot them.\n", "loss_values = []\n", "\n", "# For each epoch...\n", "for epoch_i in range(0, epochs):\n", "\n", " # ========================================\n", " # Training\n", " # ========================================\n", "\n", " # Perform one full pass over the training set.\n", "\n", " print(\"\")\n", " print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n", " print('Training...')\n", "\n", " # Measure how long the training epoch takes.\n", " t0 = time.time()\n", "\n", " # Reset the total loss for this epoch.\n", " total_loss = 0\n", "\n", " # Put the model into training mode. Don't be mislead--the call to\n", " # `train` just changes the *mode*, it doesn't *perform* the training.\n", " # `dropout` and `batchnorm` layers behave differently during training\n", " # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n", " model.train()\n", "\n", " # For each batch of training data...\n", " for step, batch in enumerate(train_dataloader):\n", "\n", " # Progress update every 40 batches.\n", " if step % 40 == 0 and not step == 0:\n", " # Calculate elapsed time in minutes.\n", " elapsed = format_time(time.time() - t0)\n", "\n", " # Report progress.\n", " print(' Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n", "\n", " # Unpack this training batch from our dataloader.\n", " #\n", " # As we unpack the batch, we'll also copy each tensor to the GPU using the\n", " # `to` method.\n", " #\n", " # `batch` contains three pytorch tensors:\n", " # [0]: input ids\n", " # [1]: attention masks\n", " # [2]: labels\n", " b_input_ids = batch[0].to(device)\n", " b_input_mask = batch[1].to(device)\n", " b_labels = batch[2].to(device)\n", "\n", " # Always clear any previously calculated gradients before performing a\n", " # backward pass. PyTorch doesn't do this automatically because\n", " # accumulating the gradients is \"convenient while training RNNs\".\n", " # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n", " model.zero_grad()\n", "\n", " # Perform a forward pass (evaluate the model on this training batch).\n", " # This will return the loss (rather than the model output) because we\n", " # have provided the `labels`.\n", "\n", " outputs = model(b_input_ids,\n", " token_type_ids=None,\n", " attention_mask=b_input_mask,\n", " labels=b_labels)\n", "\n", " # The call to `model` always returns a tuple, so we need to pull the\n", " # loss value out of the tuple.\n", " loss = outputs[0]\n", "\n", " # Accumulate the training loss over all of the batches so that we can\n", " # calculate the average loss at the end. `loss` is a Tensor containing a\n", " # single value; the `.item()` function just returns the Python value\n", " # from the tensor.\n", " total_loss += loss.item()\n", "\n", " # Perform a backward pass to calculate the gradients.\n", " loss.backward()\n", "\n", " # Clip the norm of the gradients to 1.0.\n", " # This is to help prevent the \"exploding gradients\" problem.\n", " torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n", "\n", " # Update parameters and take a step using the computed gradient.\n", " # The optimizer dictates the \"update rule\"--how the parameters are\n", " # modified based on their gradients, the learning rate, etc.\n", " optimizer.step()\n", "\n", " # Update the learning rate.\n", " scheduler.step()\n", "\n", " # Calculate the average loss over the training data.\n", " avg_train_loss = total_loss / len(train_dataloader)\n", "\n", " # Store the loss value for plotting the learning curve.\n", " loss_values.append(avg_train_loss)\n", "\n", " print(\"\")\n", " print(\" Average training loss: {0:.2f}\".format(avg_train_loss))\n", " print(\" Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n", "\n", " # ========================================\n", " # Validation\n", " # ========================================\n", " # After the completion of each training epoch, measure our performance on\n", " # our validation set.\n", "\n", " print(\"\")\n", " print(\"Running Validation...\")\n", "\n", " t0 = time.time()\n", "\n", " # Put the model in evaluation mode--the dropout layers behave differently\n", " # during evaluation.\n", " model.eval()\n", "\n", " # Tracking variables\n", " eval_loss, eval_accuracy = 0, 0\n", " nb_eval_steps, nb_eval_examples = 0, 0\n", "\n", " # Evaluate data for one epoch\n", " for batch in validation_dataloader:\n", "\n", " # Add batch to GPU\n", " batch = tuple(t.to(device) for t in batch)\n", "\n", " # Unpack the inputs from our dataloader\n", " b_input_ids, b_input_mask, b_labels = batch\n", "\n", " # Telling the model not to compute or store gradients, saving memory and\n", " # speeding up validation\n", " with torch.no_grad():\n", "\n", " # Forward pass, calculate logit predictions.\n", " # This will return the logits rather than the loss because we have\n", " # not provided labels.\n", " # token_type_ids is the same as the \"segment ids\", which\n", " # differentiates sentence 1 and 2 in 2-sentence tasks.\n", " outputs = model(b_input_ids,\n", " token_type_ids=None,\n", " attention_mask=b_input_mask)\n", "\n", " # Get the \"logits\" output by the model. The \"logits\" are the output\n", " # values prior to applying an activation function like the softmax.\n", " logits = outputs[0]\n", "\n", " # Move logits and labels to CPU\n", " logits = logits.detach().cpu().numpy()\n", " label_ids = b_labels.to('cpu').numpy()\n", "\n", " # Calculate the accuracy for this batch of test sentences.\n", " tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n", "\n", " # Accumulate the total accuracy.\n", " eval_accuracy += tmp_eval_accuracy\n", "\n", " # Track the number of batches\n", " nb_eval_steps += 1\n", "\n", " # Report the final accuracy for this validation run.\n", " print(\" Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n", " print(\" Validation took: {:}\".format(format_time(time.time() - t0)))\n", "\n", "print(\"\")\n", "print(\"Training complete!\")" ], "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", "======== Epoch 1 / 4 ========\n", "Training...\n", " Batch 40 of 369. Elapsed: 0:00:17.\n", " Batch 80 of 369. Elapsed: 0:00:33.\n", " Batch 120 of 369. Elapsed: 0:00:48.\n", " Batch 160 of 369. Elapsed: 0:01:02.\n", " Batch 200 of 369. Elapsed: 0:01:17.\n", " Batch 240 of 369. Elapsed: 0:01:32.\n", " Batch 280 of 369. Elapsed: 0:01:47.\n", " Batch 320 of 369. Elapsed: 0:02:02.\n", " Batch 360 of 369. Elapsed: 0:02:17.\n", "\n", " Average training loss: 1.32\n", " Training epcoh took: 0:02:20\n", "\n", "Running Validation...\n", " Accuracy: 0.37\n", " Validation took: 0:00:05\n", "\n", "======== Epoch 2 / 4 ========\n", "Training...\n", " Batch 40 of 369. Elapsed: 0:00:15.\n", " Batch 80 of 369. Elapsed: 0:00:30.\n", " Batch 120 of 369. Elapsed: 0:00:45.\n", " Batch 160 of 369. Elapsed: 0:00:59.\n", " Batch 200 of 369. Elapsed: 0:01:14.\n", " Batch 240 of 369. Elapsed: 0:01:29.\n", " Batch 280 of 369. Elapsed: 0:01:44.\n", " Batch 320 of 369. Elapsed: 0:01:59.\n", " Batch 360 of 369. Elapsed: 0:02:14.\n", "\n", " Average training loss: 1.32\n", " Training epcoh took: 0:02:17\n", "\n", "Running Validation...\n", " Accuracy: 0.37\n", " Validation took: 0:00:05\n", "\n", "======== Epoch 3 / 4 ========\n", "Training...\n", " Batch 40 of 369. Elapsed: 0:00:15.\n", " Batch 80 of 369. Elapsed: 0:00:30.\n", " Batch 120 of 369. Elapsed: 0:00:45.\n", " Batch 160 of 369. Elapsed: 0:00:59.\n", " Batch 200 of 369. Elapsed: 0:01:14.\n", " Batch 240 of 369. Elapsed: 0:01:29.\n", " Batch 280 of 369. Elapsed: 0:01:44.\n", " Batch 320 of 369. Elapsed: 0:01:59.\n", " Batch 360 of 369. Elapsed: 0:02:14.\n", "\n", " Average training loss: 1.31\n", " Training epcoh took: 0:02:17\n", "\n", "Running Validation...\n", " Accuracy: 0.37\n", " Validation took: 0:00:05\n", "\n", "======== Epoch 4 / 4 ========\n", "Training...\n", " Batch 40 of 369. Elapsed: 0:00:15.\n", " Batch 80 of 369. Elapsed: 0:00:30.\n", " Batch 120 of 369. Elapsed: 0:00:45.\n", " Batch 160 of 369. Elapsed: 0:01:00.\n", " Batch 200 of 369. Elapsed: 0:01:15.\n", " Batch 240 of 369. Elapsed: 0:01:30.\n", " Batch 280 of 369. Elapsed: 0:01:44.\n", " Batch 320 of 369. Elapsed: 0:01:59.\n", " Batch 360 of 369. Elapsed: 0:02:14.\n", "\n", " Average training loss: 1.31\n", " Training epcoh took: 0:02:18\n", "\n", "Running Validation...\n", " Accuracy: 0.37\n", " Validation took: 0:00:05\n", "\n", "Training complete!\n" ] } ] }, { "cell_type": "markdown", "metadata": { "id": "1-G03mmwH3aI" }, "source": [ "Let's take a look at our training loss over all batches:" ] }, { "cell_type": "code", "metadata": { "id": "68xreA9JAmG5", "colab": { "base_uri": "https://localhost:8080/", "height": 592 }, "outputId": "93bb9d6a-eb49-4dd7-9bcb-6bd17294804e" }, "source": [ " import matplotlib.pyplot as plt\n", "# % matplotlib inline\n", "\n", "import seaborn as sns\n", "\n", "# Use plot styling from seaborn.\n", "sns.set(style='darkgrid')\n", "\n", "# Increase the plot size and font size.\n", "sns.set(font_scale=1.5)\n", "plt.rcParams[\"figure.figsize\"] = (12,6)\n", "\n", "# Plot the learning curve.\n", "plt.plot(loss_values, 'b-o')\n", "\n", "# Label the plot.\n", "plt.title(\"Training loss\")\n", "plt.xlabel(\"Epoch\")\n", "plt.ylabel(\"Loss\")\n", "\n", "plt.show()" ], "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "
" ], "image/png": "iVBORw0KGgoAAAANSUhEUgAABCIAAAI/CAYAAACiSXgsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACTAUlEQVR4nOzdeVjVdf7+8fssgMBhF3BFcAP3Hc1Ky6zRNq3MMivNsqZtZirnV9aUmpU208x8m2aatFLTMtuzNFs0szIFFNwAN9zZkX2Hc87vD5Nk3Dh64LA8H9c1V5zP+SyvY68hvHkvBrvdbhcAAAAAAEADMLq6AAAAAAAA0HIQRAAAAAAAgAZDEAEAAAAAABoMQQQAAAAAAGgwBBEAAAAAAKDBEEQAAAAAAIAGQxABAAAAAAAaDEEEAAAAAABoMAQRAAAAAACgwRBEAAAAl4iJiVFkZKQiIyOdfu9PP/1UkZGRGjVqlNPvXd+eeuopRUZG6qmnnnJ1KQAA1AuzqwsAAAD152L+kj9v3jzdfPPNTqwGAACAIAIAgGatdevWZzxeWlqq0tLSc57TqlWreqtLkjw9PRUREVEv9/bx8VFERIRCQ0Pr5f4AAODCEUQAANCMbdy48YzHX3vtNf373/8+5zn1rW/fvvr666/r5d5XX321rr766nq5NwAAuDisEQEAAAAAABoMIyIAAMBpTq4tsXTpUnXt2lULFy7UDz/8oIyMDJWXl2vPnj2SpLKyMq1bt04//vij9uzZo8zMTBUXF8vf3199+/bVbbfdppEjR57xGTExMbr77rslqeZ+J3366aeaOXOm2rdvr++//167du3Sm2++qa1btyo/P1+hoaEaPXq0HnroIfn5+Z127/+9/lQnR4NER0dr2bJl2rRpkxYvXqwdO3aopKREHTp00HXXXafp06fLw8PjrH9Ga9eu1dKlS5WUlCSr1aqOHTvqhhtu0NSpU/XGG2/UeoazxcTE6L333lNCQoLy8vLk7e2tqKgo3XjjjRo/frxMJtMZr9u+fbuWLl2qhIQEZWdny2QyKSAgQO3bt9cll1yiW265RW3atKl1TUpKipYsWaLY2FhlZGTIZrMpMDBQoaGhGjZsmMaNG6cuXbo4/TMCAJovgggAAHBWR44c0eOPP66cnBx5eHjIbK79o8OaNWs0c+ZMSZLBYJDFYpHZbFZ2drbWrVundevWadq0aXryyScvuIYvv/xSM2fOVFVVlXx8fGS1WnXs2DEtWbJEGzdu1AcffCBvb+8Luvdbb72lV155RdKJdSWqqqp04MABvfbaa4qNjdXixYvP+Jf6l19+WYsWLap57evrq5SUFL3yyivasGGDBg0adGEftg7mzZunJUuWSDrxZ+7j46OioiJt3rxZmzdv1hdffKH//Oc/slgsta777LPPNHPmTNntdkmSu7u7TCaT0tLSlJaWpri4OLVt27bWAqUbN27U73//e1VWVkqS3Nzc5OnpqYyMDGVkZGj79u1yc3PTo48+Wm+fFwDQ/DA1AwAAnNVLL70kHx8fLVmyRNu2bVN8fHytdR18fX01bdo0LV++XAkJCdqyZYu2bdumn376SY8++qjc3Ny0aNEirVu37oKen5ubq6efflrjx4/XDz/8oC1btig+Pl7PPfec3NzctG/fPr311lsXdO/du3fr73//u+6//3798ssviouL05YtW/Twww9LOjHq4LPPPjvtutWrV9eEENdff71+/PFHxcXFKT4+XnPnztWOHTv0/vvvX1BN5/Puu+/WhBC33Xabfvrpp5q6Z86cKbPZrM2bN+vZZ5+tdV1ZWZnmzp0ru92uG2+8Ud9995127typrVu3KiEhQZ988onuvfdeBQUF1bpu9uzZqqys1GWXXaYvv/xSu3btUlxcnHbs2KFVq1bp0UcfVfv27evlswIAmi9GRAAAgLMyGo1asmRJreH6p+50MXr0aI0ePfq060JCQvTII4/I09NTf/3rX7Vs2TJdddVVDj+/rKxMN910k1544YWaY56enpo8ebKOHj2qxYsXa/Xq1frjH//o8L0LCwv1yCOP1PptvsVi0R/+8Aft27dP3377rVavXq0JEybUvG+32/Xqq69Kki699FK98sorMhgMkiQPDw9NnDhRZrO5ZpSIM5WXl+u1116TdCIAef7552ve8/Ly0tSpU2UymfTCCy/oq6++0r333qvevXtLkvbt26eSkhJ5eXlp3rx5tUa2eHl5qXfv3jXnnnT8+HEdOXJE0olRGCEhITXveXh4qFu3burWrZvTPycAoPljRAQAADircePGnbZmgCOuuOIKSdK2bdtktVov6B4PPvjgGY+fDDYOHz6ssrIyh+/r7u6uadOmnfPe/7t2RXJysg4fPixJeuCBB2pCiFPddNNNateuncP1nM/GjRuVn58vSXrkkUfOeM4dd9yh4OBgSdKqVatqjvv4+EiSqqqqau5xPt7e3jIaT/yomJ2dfYFVAwBwOoIIAABwVgMHDjzvOTk5OfrXv/6l2267TUOHDlXPnj0VGRmpyMhIXXvttZJOjGwoKChw+Pn+/v7q1KnTGd879Tf0hYWFDt+7W7duZ11b4uS9/7fmxMRESSfWShgwYMAZrzUYDBoyZIjD9ZzPrl27JElt27atNSrlVCaTScOGDat1viSFhYWpc+fOqqqq0sSJE7Vw4UIlJyefMxxq1aqVLrnkEknSfffdp1dffVXbt2+vWS8CAIALRRABAADO6n/XDPhfCQkJGjt2rP7zn/9o27Ztys/Pl4eHh4KCgtS6dWsFBATUnHshoxbOtQjlqYtIVlVV1cu9q6urax3Py8uTdCIgcXd3P+v1oaGhDtdzPsePH6/TvU+OYDl5vnTi8/zzn/9Uhw4dlJqaqr///e8aP368Bg0apHvuuUfLly8/47+fF154QVFRUcrNzdXrr7+uiRMnauDAgZo0aZLeeuutOo+uAADgVKwRAQAAzurk0Pwzqa6u1hNPPKHCwkL16NFDjz32mAYNGlRrt4YjR47o6quvlqSa3RrgGlFRUVqzZo1++OEH/fzzz0pISNC+ffv0yy+/6JdfftHChQu1YMGCmq1bJaldu3b67LPPtHHjRm3YsEHx8fHas2eP4uPjFR8fr4ULF+rVV1+tGTkBAEBdEEQAAIALsm3bNqWmpspkMmnBggVn/E19c1tb4OQIj/z8fFVWVp51VERmZqbTn31ydEpGRsY5zzv5/plGs7i7u+uaa67RNddcI+nECI9vvvlG//znP5Wenq6nnnrqtJ1CjEajLr/8cl1++eWSpOLiYq1fv17/+Mc/lJaWphkzZmj9+vXnHCECAMCpmJoBAAAuSHp6uiQpMDDwrNMFNm3a1JAl1btevXpJOjEVJCEh4Yzn2O12bdmyxenPPrmrRUZGhg4ePHjGc6xWq2JiYiRJffr0Oe89AwICdPvtt2vGjBmSpKSkpJrpJ2djsVh0ww036MUXX5R0Yo2QvXv31vlzAABAEAEAAC7IyZ0YcnJylJOTc9r7GRkZWrZsWUOXVa969OhRs3jmwoULzzjdZOXKlUpNTXX6sy+99FL5+/tLkv7973+f8ZwVK1YoKytLknTdddfVHD/fApMeHh41X5+cjnMh1wAAUBf8VwMAAFyQQYMGycvLS3a7XX/6059qfktvtVr1008/6a677nJxhc5nMBj06KOPSpJ+/vlnPfnkkzXTMCoqKvTRRx9p1qxZ8vPzc/qzW7VqVfPsVatW6bnnnqsJgMrKyrR06VLNmzdPknTttdfWjKCQpNWrV+v222/XihUrdPTo0ZrjJ/9d/f3vf5ckDRgwoKb2hIQE3XDDDVqyZIlSUlJks9kknRjxER8fr9mzZ0s6sTjmqetKAABwPqwRAQAALoiPj4/+3//7f5o9e7bi4uI0ZswYeXl5yWq1qqKiQgEBAZo3b54efPBBV5fqVDfccIN27typd955RytXrtQXX3whX19flZaWqqqqSsOGDVO/fv20YMECp6+bcOedd+ro0aNasmSJPvjgA3344Yfy9fVVSUlJzQ4fQ4cO1dy5c2tdZ7fblZCQUDOdxN3dXV5eXiosLKwJGEJCQmqmW5y0d+9ezZs3T/PmzZObm5u8vb1VXFxc8yyLxaK///3vtXYwAQDgfAgiAADABZs0aZLatWunt956S7t27ZLValVoaKhGjhyp6dOnX9C2mk3B008/rSFDhmjp0qVKSkpSZWWlOnfurHHjxmnKlCmaP3++JMnX19fpz545c6auvPJKLV++XPHx8crPz5e3t7eioqI0btw4jR8//rRgYNSoUXr55ZcVExOjpKQkZWdnq6CgQN7e3oqIiNCVV16pO++8s1a9ffr00f/93/8pJiZGO3bsUFZWlvLz8+Xu7q5u3brp0ksv1d13310vW5UCAJo3g529tAAAAJzq9ttvV0JCgv7whz/o4YcfdnU5AAA0KqwRAQAA4ESxsbE1UyBObnkJAAB+QxABAADgoDlz5ujTTz9VdnZ2zc4ZhYWFWrFihR566CFJ0rBhw9S3b19XlgkAQKPE1AwAAAAHjRs3Trt375Z0YuFHT09PFRYW1oQSXbt21aJFi1g/AQCAMyCIAAAAcNC6deu0du1a7dixQzk5OSouLpbFYlHXrl119dVX67bbbpOnp6erywQAoFEiiAAAAAAAAA2GNSIAAAAAAECDIYgAAAAAAAANxuzqAlB/7Ha7bLbGP/PGaDQ0iTrReNAzcBQ9A0fRM3AE/QJH0TNwVFPpGaPRIIPBcN7zCCKaMZvNrtzcEleXcU5ms1EBAd4qLCxVdbXN1eWgCaBn4Ch6Bo6iZ+AI+gWOomfgqKbUM4GB3jKZzh9EMDUDAAAAAAA0GIIIAAAAAADQYAgiAAAAAABAgyGIAAAAAAAADYYgAgAAAAAANBiCCAAAAAAA0GAIIgAAAAAAQIMhiAAAAAAAAA2GIAIAAAAAADQYgggAAAAAANBgCCIAAAAAAECDIYgAAAAAAAANhiACAAAAAAA0GIIIAAAAAADQYMyuLgAtl81mV/KhXFUdzJObwa4u7fxkNBpcXRYAAAAAoB4RRMAltu7J0vK1+5RXVFFzLMDHQ3eM7qZBkSEurAwAAAAAUJ+YmoEGt3VPlv7z2a5aIYQk5RVV6D+f7dLWPVkuqgwAAAAAUN8IItCgbDa7lq/dd85z3l+7TzabvYEqAgAAAAA0JIIINKi9R/NPGwnxv3KLKrT3aH7DFAQAAAAAaFAEEWhQ+SXnDiEcPQ8AAAAA0LQQRKBB+Xt7OPU8AAAAAEDTQhCBBtW9o78CfM4dMni4GdUp1KeBKgIAAAAANCSCCDQoo9GgO0Z3O+c5FVU2vbBsi45kFjVQVQAAAACAhkIQgQY3KDJED9/U+7SREYE+Hhp3abj8Le5KP16qF5Zu0XdbjspuZwcNAAAAAGguzK4uAC3ToMgQDegWrJS0AlXZDXIz2NWlnZ+MRoNGDeqgxV/t1rb9OXp/7T4lHszVtOt6yNfL3dVlAwAAAAAuEiMi4DJGo0E9wgM1cmAH9QgPlNFokCT5eLnr0Vv6aPLV3WU2GbUj5bhmLYpV0qFcF1cMAAAAALhYBBFolAwGg64a1EF/uXuQ2gZ5qaC4Un9fsU2fbEhRtdXm6vIAAAAAABeIIAKNWlioj56bMkQj+rWTXdLqTYc1/714ZeWXubo0AAAAAMAFIIhAo+fhbtLUsVF6cHxveXqYdSCtULMXxWpzUoarSwMAAAAAOIggAk3GkKgQzZk2RF3b+6m80qqFXyTp7dVJKq+sdnVpAAAAAIA6IohAk9Laz1NPTh6gG4aHyyBp484MzVkcp8MZRa4uDQAAAABQBwQRaHJMRqNuGtFZf540QAE+HsrMK9MLS7fo29gjstntri4PAAAAAHAOBBFosqI6BWjOtGgN6NZaVptdK77fr1c/2qHCkkpXlwYAAAAAOAuCCDRpFk83PXJzH911TXeZTUbtPHBczy2KVeLBXFeXBgAAAAA4A4IINHkGg0FXDuyg56YMVvvW3iosqdTfP9imj9bvV7XV5uryAAAAAACnIIhAs9EhxKK/TBmsKwa0lyStiTmiee9uVVZeqYsrAwAAAACcRBCBZsXDzaS7fxeph2/qLe9WZh1ML9KsxXHatCvD1aUBAAAAAEQQgWZqUGSIZt8Tre4d/FRRadWbq5L05pdJKquodnVpAAAAANCiEUSg2Qrya6U/3zFA4y6LkMEgbUrM0JwlcTqYXujq0gAAAACgxSKIQLNmMho17rIIPXnHQAX6eigrr0wvLduqr2OOyGa3u7o8AAAAAGhxCCLQInTv6K8506I1KDJYVptdH67fr39+uF0FxRWuLg0AAAAAWhSCCLQY3q3c9ND43rp7TKTczUYlHszVrEWx2nnguKtLAwAAAIAWgyACLYrBYNAV/dvr2alD1CHYW4WlVfrnh9u1Yt0+VVXbXF0eAAAAADR7BBFokdq39tZf7h6sUQPbS5K+jTuql5ZtVUZuqYsrAwAAAIDmjSACLZa7m0l3XhOpR2/uI+9WZh3OLNKcxXHauDNddhayBAAAAIB6QRCBFm9A92DNmRatqDB/VVRZ9fbqZL35ZZLKKqpdXRoAAAAANDsEEYCkQN9WmnH7AN00orOMBoM2J2Vq9uJYpaQVuLo0AAAAAGhWCCKAXxmNBt0wPFxPTR6oIN9Wys4v1/x34/XV5sOyMVUDAAAAAJyCIAL4H107+GnOtCEaHBUiq82uj39I0T8+2Kb84gpXlwYAAAAATR5BBHAGXq3c9OC4Xpo6NkrubkYlHcrTc2/Havv+HFeXBgAAAABNGkEEcBYGg0Ej+rXTc1OGqGOIRcVlVXr14x1avnavqqptri4PAAAAAJokggjgPNq19tZf7h6k0YM6SJLWbjmmF5duUfrxEhdXBgAAAABND0EEUAduZpPuuLq7/jChryyebjqSVaw5S+L00/Y02VnIEgAAAADqjCACcED/rq01Z1q0enQKUGWVTYvX7NaCLxJVWl7l6tIAAAAAoEkgiAAcFODjoSdu669bRnaW0WBQbHKWZi+O0/7UAleXBgAAAACNntnVBdRVdna2Nm7cqF27dmnnzp1KTk5WRUWFoqOjtWzZsgu655o1a/TLL78oMTFRWVlZys/Pl5ubm8LDwzVy5EhNmTJFAQEBp12Xl5entWvX1lybnp4uo9Gotm3b6rLLLtPUqVPVoUOH066z2+1KSEjQ999/r61bt+rAgQMqLi6Wj4+PevbsqfHjx+uGG26QwWC4oM+DhmM0GnTdJeGKCgvQgi8SlVNQrvnvxmvc5RG6blgnGY38OwQAAACAMzHYm8gE9yVLlmjevHmnHb+YIGLcuHHavXu33N3dFRwcrICAAOXm5iotLU2SFBQUpEWLFikqKqrWdXfccYe2bt0qSfL29lZYWJjKy8t19OhRVVdXy8vLS6+++qpGjBhR67pNmzZp6tSpNa87duwoX19fpaamKj8/X5J0xRVX6LXXXpO7u/sFfaZTWa025eY27gUVzWajAgK8lZdXouomuhNFaXm1ln27RzFJmZKkqDB/Tb+hlwJ8PFxcWfPUHHoGDYuegaPoGTiCfoGj6Bk4qin1TGCgt0ym80+8aDIjIiwWi4YPH64+ffqoT58+SkpK0uuvv35R95w8ebIiIiLUv39/ubm51Rzfs2ePZsyYob179+qJJ57Q6tWra11nMpl0/fXX6/bbb9fAgQNlMpkkSRkZGXr66ae1ceNGPfbYY/rmm2/UunXrmuvsdrs6dOigKVOm6LrrrlNQUFDNe59//rmeffZZ/fDDD3r11Vf15z//+aI+GxqOVyuz7r+hp3qFB+q97/Zq95F8zVoUq2nX9lD/bq3PfwMAAAAAaEGazIiI//Xuu+9q7ty5FzUi4lx27NihW2+9VZL01VdfqUuXLjXv5eXlnXHKhiQVFRXpmmuuUW5urmbOnFlrBERxcbE8PDxqhR6neuONN/TPf/5T/v7+2rRpk4zGi1vCgxERDS8jt1RvrNylI5nFkqSrBnbQxFFd5GY2ubiy5qO59QzqHz0DR9EzcAT9AkfRM3BUU+qZuo6IYLHKs+jcuXPN12VlZbXeO1sIIUk+Pj7q37+/JOngwYO13rNYLGcNISTVTOXIz89Xbm6uoyWjEWgT6KVn7hqsa4Z0lCStiz+mue9sVWpO4w6EAAAAAKChEEScxck1ILy8vBQREeHQtRUVFZIkT09Ph64rLy+v+bpVq1YOXYvGw81s1O1XddOfbu0nHy83Hcsu1twlcdqwLVVNdAASAAAAADgNQcQpbDabMjMz9emnn2rmzJmSpBkzZsjb27vO98jMzFRsbKwkafDgwQ49/+RaFFFRUbJYLA5di8anb5cgPT8tWr3CA1RZbdM7X+/Rfz/fpZLyKleXBgAAAAAu02QWq6xPZ9qRo2/fvpo/f/5pO1+cz9y5c1VVVaWuXbvqyiuvrPN1u3bt0ooVKyRJ999/v0PPPBezuXFnTSfnD9VlHlFTFOTvqT9PHqg1mw/r4/Up2rInWwfTi/TgTb3VvaO/q8trkpp7z8D56Bk4ip6BI+gXOIqegaOaY88QREgKDQ3VwIEDZbValZaWppycHCUnJ2vlypXq37+/fH1963SfhQsX6rvvvpObm5vmz59fs5vG+eTk5OjRRx9VdXW1rr76al133XUX83FqGI0GBQTUfTSHK/n6OjaNpam589peiu7dTq+8u1Xpx0v00tItuv2aKE0c3V0mo8HV5TVJzb1n4Hz0DBxFz8AR9AscRc/AUc2pZwgiJI0dO1Zjx46teb17927NnTtXq1atUkpKij755JPzhgqfffaZ/vGPf8hgMOjFF19Unz596vTsoqIiTZ8+XWlpaerVq5fmz59/UZ/lVDabXYWFpU67X30wmYzy9fVUYWGZrNbGvQLsxQr2cdfsaUP0zprd+mVXhpZ/s1tbkzP0+3G9FeTHmiB11ZJ6Bs5Bz8BR9AwcQb/AUfQMHNWUesbX17NOIzcIIs4gKipKCxYs0OjRo5WcnKzVq1frxhtvPOv5X331lZ555hnZ7XbNnj1b48aNq9NzSkpKdN999ykpKUndunXT22+/7fS1IRr79i4nWa22JlPrxXAzGXXf9T3VMzxAy77dqz1H8vWXNzdr6tgeGhQZ7OrympSW0jNwHnoGjqJn4Aj6BY6iZ+Co5tQzzWeSiZNZLBZFR0dLkhITE8963nfffac///nPslqtevLJJzVp0qQ63b+srEwPPPCAtm3bpvDwcC1evPic24KieRneu61m3zNE4W18VFJerf98tlPLvtmjyiqrq0sDAAAAgHpFEHEO1dXVkiSr9cx/OdywYYMee+wxVVdX6w9/+IOmTZtWp/tWVFTowQcfVFxcnNq3b68lS5YoOJjfhrc0oQFeevquQRozNEyStD4hVXPf2aJj2cUurgwAAAAA6g9BxFnk5+fXbMPZo0eP097ftGmTHn30UVVVVemBBx7Qww8/XKf7VlVV6dFHH9WmTZsUGhqqd955R23btnVq7Wg6zCajJl7ZVY/f1k++3u5KzSnR3He2aH1Cqux2u6vLAwAAAACna/ZBxKRJkzRq1CgtWbKk1vHY2Fi9/vrrOnbs2GnXJCYm6t5771VRUZFCQ0M1ZsyYWu8nJCTooYceUkVFhaZOnarHH3+8TrVYrVY98cQT2rBhg4KDg/XOO++oY8eOF/zZ0Hz0jgjSnGnR6t05UFXVNi37Zo/+89kuFZdVubo0AAAAAHCqJrNYZXp6usaPH1/zurKyUpIUHx+voUOH1hy/7777NH369JrXmZmZSk1NVVFRUa37FRYW6tVXX9Wrr76q4OBghYSEyGQyKT09XdnZ2ZJObOu5YMECeXvX3gLzqaeeUmlpqUwmk3bs2HHWdSF69uypZ599tub1mjVr9M0330iS3N3d9fTTT5/18z777LPq2bPnuf5I0Mz4ebvrT7f203dxR/XxDymK35utg+mFuv+GnooMY/0QAAAAAM1DkwkirFar8vPzTzteXV1d63h5eXmd7jdgwADNnDlTMTEx2r9/vw4dOqTKykr5+vpq6NChGjVqlCZMmHDGXSyqqqpqaoqPjz/rM8zm2n+8J8MTSUpNTVVqaupZr/3f4AQtg9Fg0O+iwxQZ5q8FKxOVmVemv76foBuGh+uGS8NlMjb7QUwAAAAAmjmDnYnozZbValNubomryzgns9mogABv5eWVNJutaJylvLJa7323Vxt3ZkiSunbw0/039FRrP08XV+Za9AwcRc/AUfQMHEG/wFH0DBzVlHomMNBbJtP5f3nKr1eBRqqVu1n3XtdT99/QU63cTdp/rECzF8Vpy+4sV5cGAAAAABeMIAJo5Ib1aqPZ06IV0dZXpRXVev3zXXrn692qqDrztrIAAAAA0JgRRABNQIi/p2beOVDXDuskg6QN29L0/JI4Hc0qdnVpAAAAAOAQggigiTCbjJpwRRc9fnt/+Xm7K/14qea+s0Xrth4TS70AAAAAaCoIIoAmpld4oObcG62+XYJUbbXpve/26rVPdqqotPL8FwMAAACAixFEAE2Qr5e7/jihryZd1U1mk0Hb9udo1qJYJR/Oc3VpAAAAAHBOBBFAE2UwGHT1kI76y92D1SbQS/nFlXrl/QR9+mOKqq2Ne1sfAAAAAC0XQQTQxIWF+mjW1CG6vG9b2SWt+uWwXl4er5z8MleXBgAAAACnIYgAmgEPd5PuubaHfj+ulzw9TEpJLdSsxXGKTc50dWkAAAAAUAtBBNCMRPcI1Zx7otWlna/KKqr1xspELf4qWRWVVleXBgAAAACSCCKAZqe1v6eenDxQ1w/vJIOkn3aka86SOB3JLHJ1aQAAAABAEAE0R2aTUTeP6KIZkwbI3+KujNxSvbB0i77bclR2u93V5QEAAABowQgigGasR6cAzZkWrf5dW6vaatf7a/fp1Y93qLC00tWlAQAAAGihCCKAZs7Hy12P3tJHk6/uLrPJqB0pxzVrUaySDuW6ujQAAAAALRBBBNACGAwGXTWog56dMlhtg7xUUFypv6/Ypo9/SFG11ebq8gAAAAC0IAQRQAvSMcSi56YO0cj+7WSX9NXmw5r3bryy8stcXRoAAACAFoIgAmhhPNxMmjImSg+N7y0vD7MOphdq9qJYbU7KcHVpAAAAAFoAggighRocFaLZ04aoawc/lVdatfCLJL29OknlldWuLg0AAABAM0YQAbRgrf089eQdA3TjpeEyGKSNOzM0Z3GcDmcUubo0AAAAAM0UQQTQwpmMRo2/vLP+36QBCvDxUGZemV5YukXfxB6RzW53dXkAAAAAmhmCCACSpMiwAM2ZFq2B3YNltdn1wff79X8fbVdBSaWrSwMAAADQjBBEAKhh8XTTwzf11l2/i5Sb2ahdB3I1a1Gsdh087urSAAAAADQTBBEAajEYDLpyQHs9O2Ww2rf2VmFJpf7xwXZ9uH6/qq02V5cHAAAAoIkjiABwRh2CLXp2ymBdOaC9JOnrmCN6adlWZeaVurgyAAAAAE0ZQQSAs3J3M+mu30Xq4Zv6yLuVWYcyijR7cZw27cpwdWkAAAAAmiiCCADnNSgyWHOmRat7R39VVFr15qokvfllksoqql1dGgAAAIAmhiACQJ0E+rbS/5s0QOMvj5DBIG1KzNCcxXE6mF7o6tIAAAAANCEEEQDqzGg06MZLI/TkHQMV5OuhrPwyvbRsq9bEHJbNbnd1eQAAAACaAIIIAA7r3tFfs6dFa3BksKw2uz5an6J/frBNBcUVri4NAAAAQCNHEAHggni3ctOD43tryphIuZuNSjyUp+cWxWpHynFXlwYAAACgESOIAHDBDAaDRvZvr2enDlGHYG8VlVbp/z7arhXr9qmq2ubq8gAAAAA0QgQRAC5a+9beenbKYF01sIMk6du4o3pp2VZl5Ja6uDIAAAAAjQ1BBACncDObNPma7nr0lj6yeLrpcGaR5iyO08870mVnIUsAAAAAvyKIAOBUA7oFa860aEWF+auiyqpFXyVr4ZdJKquodnVpAAAAABoBgggAThfg46EZtw/QzSM6y2gwKCYpU7MWxSolrcDVpQEAAABwMYIIAPXCaDTo+uHheurOgQrybaWcgnLNfzdeqzcdko2pGgAAAECLRRABoF51be+nOdOGKLpHiKw2uz7ZcEB/X7FNeUUVri4NAAAAgAsQRACod16t3PTAjb10z9goubsZlXw4T7MWxWrb/hxXlwYAAACggRFEAGgQBoNBl/drp1lThygsxKLisir96+MdWv7dXlVV21xdHgAAAIAGQhABoEG1DfLWM3cP1tWDO0qS1m49pheWblH68RIXVwYAAACgIRBEAGhwbmajJo3upj9O6CuLp5uOZhVrzpI4/bg9TXYWsgQAAACaNYIIAC7Tr2trPX9vtHp0ClBllU1L1uzWGysTVVpe5erSAAAAANQTgggALuVv8dATt/fXhCu6yGQ0KG53lmYtitP+YwWuLg0AAABAPSCIAOByRoNB1w7rpKfuHKjWfq10vLBc89+L15e/HJLNxlQNAAAAoDkhiADQaHRp56c506I1tGeobHa7PvvxgF5ZkaC8ogpXlwYAAADASQgiADQqnh5m3X9DT917XQ95uJm0+0i+nns7Rgn7sl1dGgAAAAAnIIgA0OgYDAZd2qetZt0zRJ1CfVRSXq3XPtmp977dq8pqq6vLAwAAAHARCCIANFptAr309F2D9LvojpKkdfHHNGdRnI5kFLq4MgAAAAAXiiACQKPmZjbqtlHd9NjEfvL1ctPRrGI99n8/an38MdntLGQJAAAANDUEEQCahD6dgzRnWrR6dw5UZZVVi7/ardc/36WS8ipXlwYAAADAAQQRAJoMP4uHZkwaoHuu7yWT0aCte7I1a1Gs9h7Nd3VpAAAAAOqIIAJAk2I0GHTzlV317NQhCgnwVG5hhV5eHq8vfj4om42pGgAAAEBjRxABoEnq3M5Xs6YO0fDebWS3S5//fFB/fT9BuYXlri4NAAAAwDkQRABosjw9zLrv+p6afn1PebibtPdovmYtitXWPdmuLg0AAADAWRBEAGjyLundRrPvGaKItj4qKa/Wfz7bqaXf7FFlldXVpQEAAAD4HwQRAJqF0AAvzbxzkMYODZMk/ZCQqrnvbNGx7GIXVwYAAADgVAQRAJoNs8moW6/sqsdv6ydfb3el5pRo7jtbtD7+mOx2FrIEAAAAGgOCCADNTu+IID0/LVq9OweqqtqmZd/u1b8/3anisipXlwYAAAC0eAQRAJolX293/enWfrp9VFeZjAYl7MvRrEWx2nMkz9WlAQAAAC0aQQSAZstoMOia6DD95e7BCg30Ul5Rhf76foI+/+mArDabq8sDAAAAWiSCCADNXqc2Ppo1dbAu69NWdrv0xcZDenl5gnIKylxdGgAAANDiEEQAaBFauZs17boeuv/GnvL0MGn/sQLNXhSnLbuzXF0aAAAA0KIQRABoUYb1bKNZ90SrcztflVZU6/XPd2nJmt2qqLK6ujQAAACgRSCIANDihPh76qnJA3XdJZ1kkPTj9jQ9vyRORzKLXF0aAAAA0OwRRABokcwmo24Z2UVP3N5ffhZ3pR8v1QtLt2rd1mOy2+2uLg8AAABotggiALRoPcMDNWdatPp2CVK11ab3vtur1z7ZqaLSSleXBgAAADRLBBEAWjxfL3f9cUJf3TG6m8wmg7btz9GsRbFKPpzn6tIAAACAZocgAgAkGQwGjR7cUX+5e7DaBnkpv7hSr7yfoE82pKjaanN1eQAAAECzQRABAKcIC/XRc1OGaES/trJLWr3psF5+L17Z+WWuLg0AAABoFggiAOB/eLibNHVsD/1+XC95epiVklao2YtjFZuc6erSAAAAgCbP7OoC6io7O1sbN27Url27tHPnTiUnJ6uiokLR0dFatmzZBd1zzZo1+uWXX5SYmKisrCzl5+fLzc1N4eHhGjlypKZMmaKAgIDTrsvLy9PatWtrrk1PT5fRaFTbtm112WWXaerUqerQocM5n71582YtXrxY27dvV2lpqdq1a6cxY8bo/vvvl5eX1wV9HgDOFd0jVJ3b+mrBl4lKSS3UGysTtetgriaP7i4Pd5OrywMAAACaJIO9iexTt2TJEs2bN++04xcTRIwbN067d++Wu7u7goODFRAQoNzcXKWlpUmSgoKCtGjRIkVFRdW67o477tDWrVslSd7e3goLC1N5ebmOHj2q6upqeXl56dVXX9WIESPO+Nxly5bpxRdflN1uV5s2bRQYGKj9+/ersrJSXbp00fLly+Xv739Bn+lUVqtNubklF32f+mQ2GxUQ4K28vBJVVzMPH+fnip6x2mxa+fMhrf7lkOyS2gR66YEbe6lTG58GeT4uDt9n4Ch6Bo6gX+AoegaOako9ExjoLZPp/BMvmsyICIvFouHDh6tPnz7q06ePkpKS9Prrr1/UPSdPnqyIiAj1799fbm5uNcf37NmjGTNmaO/evXriiSe0evXqWteZTCZdf/31uv322zVw4ECZTCd+M5qRkaGnn35aGzdu1GOPPaZvvvlGrVu3rnXtrl279NJLL0mSnn/+eU2cOFEGg0GZmZl68MEHlZiYqGeffVavvfbaRX02AM5jMhp184jO6tkpQG+uSlJGbqleXLZFt17RVaMHd5DBYHB1iQAAAECT0WRGRPyvd999V3Pnzr2oERHnsmPHDt16662SpK+++kpdunSpeS8vL++MUzYkqaioSNdcc41yc3M1c+ZMTZ06tdb7Dz30kNatW6fx48fr5ZdfrvXeoUOHNHbsWNlsNq1cufK0kRiOYkQEmiNX90xxWZUWf5WshH05kqS+XYI07boe8vVyb/BaUDeu7hk0PfQMHEG/wFH0DBzVlHqmriMiWKzyLDp37lzzdVlZ7dXyzxZCSJKPj4/69+8vSTp48GCt90pKSvTTTz9JkiZOnHjateHh4Ro2bJgk6euvv76gugHUL4unmx65uY/uvKa7zCajdqQc16y3Y5V0KNfVpQEAAABNAkHEWZxcA8LLy0sREREOXVtRUSFJ8vT0rHU8OTlZlZWVcnd3V9++fc947aBBgyRJ27dvd7RkAA3EYDBo1MAOem7KYLVr7a2Ckkr9fcU2ffTDflVbG3dKDQAAALgaQcQpbDabMjMz9emnn2rmzJmSpBkzZsjb27vO98jMzFRsbKwkafDgwbXeOzlCol27drXWpDhVWFhYrXMBNF4dQix6dspgXdG/neyS1mw+onnvxisrr9TVpQEAAACNVpNZrLI+nWlHjr59+2r+/Pln3fnibObOnauqqip17dpVV155Za33CgoKJEl+fn5nvf7keyfPvVhmc+POmk7OH6rLPCJAanw9YzYbNe36nurTJUhvr07WwfRCzV4cp6nX9tDw3m1cXR7U+HoGjR89A0fQL3AUPQNHNceeIYiQFBoaqoEDB8pqtSotLU05OTlKTk7WypUr1b9/f/n6+tbpPgsXLtR3330nNzc3zZ8/v2Y3jZNOTtk422gISXJ3d6917sUwGg0KCKj7aA5X8vX1PP9JwCkaW89cM7yz+vdoo7+/t1VJB3P1xue7tPdYgR64qY+8Wp39//NoOI2tZ9D40TNwBP0CR9EzcFRz6hmCCEljx47V2LFja17v3r1bc+fO1apVq5SSkqJPPvnktFDhf3322Wf6xz/+IYPBoBdffFF9+vQ57RwPDw9JUlVV1VnvU1lZWevci2Gz2VVY2LiHiJtMRvn6eqqwsExW5tajDhpzz7hJ+vOk/vri50P6/KcD+n7LUSUeOK6HbuqtiLZ1CzThfI25Z9A40TNwBP0CR9EzcFRT6hlfX886jdwgiDiDqKgoLViwQKNHj1ZycrJWr16tG2+88aznf/XVV3rmmWdkt9s1e/ZsjRs37ozn1WXaRV2mbziisW/vcpLVamsytaJxaMw9c8PwcEV29NfCLxOVmVuq5xfH6ZaRXXRNdEcZDQZXl9diNeaeQeNEz8AR9AscRc/AUc2pZ5rPJBMns1gsio6OliQlJiae9bzvvvtOf/7zn2W1WvXkk09q0qRJZz03PDxckpSWlnbWURFHjhypdS6Apql7R3/Nvidag7oHy2qz68P1+/V/H21XQUmlq0sDAAAAXIog4hyqq6slSVar9Yzvb9iwQY899piqq6v1hz/8QdOmTTvn/Xr06CE3NzdVVlZqx44dZzzn5Lah/fv3v/DCATQKFk83PXRTb939u0i5mY3adSBXsxbFatfB464uDQAAAHAZgoizyM/Pr9mGs0ePHqe9v2nTJj366KOqqqrSAw88oIcffvi897RYLLrsssskSR9++OFp7x86dEibN2+WJI0ZM+ZiygfQSBgMBl0xoL2emzJY7YO9VVhSqX98sF0ffr9f1Y18jh8AAABQH5p9EDFp0iSNGjVKS5YsqXU8NjZWr7/+uo4dO3baNYmJibr33ntVVFSk0NDQ00KBhIQEPfTQQ6qoqNDUqVP1+OOP17mehx56SAaDQStXrtQHH3wgu90uScrKytLjjz8um82m0aNHKyoqyvEPC6DRah9s0bN3D9aVA9tLkr6OPaKXlm1VZm7jXlAWAAAAcDaD/eTfhBu59PR0jR8/vuZ1ZWWlSktLZTabZbFYao7fd999mj59es3rUaNGKTU1VY888ogeffTRmuNr166tGcUQHByskJAQmUwmpaenKzs7W9KJbT0XLFhw2oiI3/3udzp06JBMJpP69et31pp79uypZ5999rTjS5Ys0fz582W329W2bVsFBARo//79qqysVEREhJYvX67AwEDH/oDOwGq1KTe35KLvU5/MZqMCAryVl1fSbBZeQf1qDj0Tvzdbi79KVkl5tTzcTbrrmu4a3rutq8tqtppDz6Bh0TNwBP0CR9EzcFRT6pnAQO/mtWuG1WpVfn7+acerq6trHS8vL6/T/QYMGKCZM2cqJiZG+/fv16FDh1RZWSlfX18NHTpUo0aN0oQJE2qFHCedXGjSarUqPj7+rM8wm8/8xzt16lRFRkZq0aJF2rFjh44fP6527dppzJgxuv/+++Xt7V2nzwCgaRrYPVjhbXz05pdJ2nM0X2+tSlbiwVzdeU2kPD2azLdlAAAA4II0mRERcBwjItAcNaeesdnsWr3pkFb+fEg2u10h/p56YFwvRbT1dXVpzUpz6hk0DHoGjqBf4Ch6Bo5qSj1T1xERzX6NCABorIxGg264NEJPTh6gIF8PZeWX6aVlW7Vm82HZyIgBAADQTBFEAICLdevgrznTojU4KkRWm10f/ZCif36wTfnFFa4uDQAAAHA6gggAaAS8WrnpwXG9NHVslNzNRiUeytOsRbHakXLc1aUBAAAATkUQAQCNhMFg0Ih+7fTc1CHqEGxRUWmV/u+j7Vqxbp+qGvl8QAAAAKCuCCIAoJFp19pbz04ZpNGDOkiSvo07qheXbVFGbqmLKwMAAAAuHkEEADRCbmaT7ri6u/5wS19ZPN10JLNYcxbH6ecd6WKzIwAAADRlBBEA0Ij179Zac6ZFKyrMXxVVVi36KlkLv0xSaXm1q0sDAAAALghBBAA0cgE+Hppx+wDdMrKzjAaDYpIyNXtxrFJSC1xdGgAAAOAwgggAaAKMRoOuuyRcM+8cqNZ+rZRTUK5578Zr9aZDsjFVAwAAAE0IQQQANCFd2vtp9j3Riu4RIpvdrk82HNDfV2xTXlGFq0sDAAAA6oQgAgCaGK9WZj1wYy/dc22U3N2MSj6cp1mLYrVtf46rSwMAAADOiyACAJogg8Ggy/u206ypQxQWalFxWZX+9fEOLf9ur6qqra4uDwAAADgrgggAaMLaBnnrmbsG65ohHSVJa7ce0wtLtyotp8TFlQEAAABnRhABAE2cm9mo26/qpj/d2lc+Xm46mlWs59+J04/b02RnIUsAAAA0MgQRANBM9O3SWnOmRatneIAqq2xasma3/rsyUaXlVa4uDQAAAKhBEAEAzYi/xUOP39Zft17RRSajQVt2Z2nWojjtP1bg6tIAAAAASQQRANDsGA0GjR3WSTPvHKRg/1Y6Xliu+e/F68uNB2WzMVUDAAAArkUQAQDNVOd2vpp9T7SG9QqVzW7XZz8d1N/eT1BuYbmrSwMAAEALRhABAM2Yp4dZ99/QS/dd30MebibtOZqvWYtilbA329WlAQAAoIUiiACAFmB477aafc8QdWrjo5Lyar326U69++0eVVZZXV0aAAAAWhiCCABoIUIDvfTMXYM0JjpMkvR9fKpeWLpFqTklLq4MAAAALQlBBAC0IGaTURNHddXjE/vJ18tNx7JLNHdJnH7Yliq7nYUsAQAAUP8IIgCgBerdOUhz7h2q3hGBqqy2aenXe/T6Z7tUXFbl6tIAAADQzBFEAEAL5eftrj9N7KeJV3aVyWjQ1r3Zmr04VnuP5ru6NAAAADRjBBEA0IIZDQaNGRqmZ+4epJAAT+UWVujl5fFa+fNBWW02V5cHAACAZoggAgCg8Da+mjV1iC7t3UZ2u7Ty54P62/IEHS8od3VpAAAAaGYIIgAAkiRPD7Puvb6npt/QU63cTdp7rECzF8dq654sV5cGAACAZoQgAgBQyyW92mj2PUMU0dZHJeXV+s9nu7T0692qqLK6ujQAAAA0AwQRAIDThAR4aeadgzR2WJgk6YdtaXrhnS06ll3s4soAAADQ1BFEAADOyGwy6tYruuqJ2/rLz9tdqTklmvvOFn0ff0x2u93V5QEAAKCJIogAAJxTr4hAzZkWrb5dglRVbdO73+7Vvz/dqeKyKleXBgAAgCaIIAIAcF6+3u7644S+mnRVN5lNBiXsy9GsRbHacyTP1aUBAACgiSGIAADUicFg0NVDOuqZuwYrNNBLeUUV+uvyBH324wFZbTZXlwcAAIAmgiACAOCQTm18NGvqYF3Wt63skr785ZBefi9BOQVlri4NAAAATQBBBADAYa3czZp2bQ89cGMveXqYtD+1QLMWxSlud5arSwMAAEAjRxABALhgQ3uGavY90erSzldlFdX67+e7tGTNblVUWl1dGgAAABopgggAwEUJ9vfUk5MH6rpLOskg6cftaXr+nTgdySxydWkAAABohAgiAAAXzWwy6paRXTTj9v7yt7gr/XipXli6VWu3HJXdbnd1eQAAAGhECCIAAE7TIzxQc6ZFq3/X1qq22rR87T699slOFZVWuro0AAAANBIEEQAAp/Lxctejt/TR5Ku7y2wyatv+HD23KFbJh3JdXRoAAAAaAYIIAIDTGQwGXTWog/5y9yC1DfJSQXGlXlmxTZ9sSFG11ebq8gAAAOBCBBEAgHoTFuqj56YM0Yh+7WSXtHrTYb38Xryy88tcXRoAAABchCACAFCvPNxNmjo2Sg+N7y0vD7NS0go1e3GsYpIyXV0aAAAAXIAgAgDQIAZHhWj2tCHq2sFPZRVWLfgiUYtWJ6ui0urq0gAAANCACCIAAA2mtZ+nnrxjgG68NFwGg/TzznTNXhKnwxlFri4NAAAADYQgAgDQoExGo8Zf3ln/b9IABfh4KDO3VC8u26Jv447Kbre7ujwAAADUM3N9P8Bqter999/Xxo0bZTQadcUVV+jWW2+t78cCABq5yLAAzZkWrcVfJSthX45WrNunpEO5mnZtD/l6u7u6PAAAANQTp4yI+Pjjj9WjRw/96U9/Ou29xx9/XC+++KJ++OEHrVu3Ts8995wee+wxZzwWANDEWTzd9MjNfXTXNd1lNhm1I+W4Zi2KVeKhXFeXBgAAgHrilCBi48aNkqTrr7++1vGYmBh98803stvtGjBggIYPHy5J+vrrr7V27VpnPBoA0MQZDAZdObCDnpsyWO1be6ugpFL/WLFNH/2wX9VWm6vLAwAAgJM5JYhITk6WJA0cOLDW8c8//1ySNHHiRC1fvlyLFi3So48+Krvdrs8++8wZjwYANBMdQiz6y5TBumJAe9klrdl8RPPe3aqsvFJXlwYAAAAnckoQkZeXJ3d3dwUGBtY6vmnTJhkMBt111101xyZPnixJ2rVrlzMeDQBoRjzcTLr7d5F6+Kbe8m5l1sH0Is1eHKdNiRmuLg0AAABO4pQgoqSkRB4eHrWOZWVlKSMjQ0FBQerWrVvNcT8/P1ksFuXmMv8XAHBmgyJDNGdatLp38FN5pVVvfpmkt1Ylqayi2tWlAQAA4CI5JYiwWCwqKipSWVlZzbG4uDhJ0oABA854zf8GFwAAnCrQt5X+3x0DNf6yCBkM0i+7MjRnSZwOZRS6ujQAAABcBKcEESdHPKxZs6bm2Oeffy6DwaAhQ4bUOreoqEjFxcVq3bq1Mx4NAGjGjEaDbrwsQk/eMVCBvh7KyivTi0u36uuYI7LZ7a4uDwAAABfA7IybXH/99YqLi9Pzzz+v7du3KycnRz/99JPc3d01duzYWucmJCRIksLDw53xaABAC9C9o7/mTIvWkjW7tXVPtj5cv19Jh3J17/U95eft7uryAAAA4ACnjIiYMGGChg8frvLycn344Ydat26dDAaD/vSnPyk4OLjWuV9//fUZR0oAAHAu3q3c9ND43rp7TKTczUbtOpirWW/HaNeB464uDQAAAA5wyogIk8mkt956S6tWrVJCQoJ8fX01YsQIDRo0qNZ5lZWVys7O1uDBgzVixAhnPBoA0IIYDAZd0b+9unXw14KVu3Qsu0T/+HC7xkSH6eaRnWU2OSVfBwAAQD0y2O1Msm2urFabcnNLXF3GOZnNRgUEeCsvr0TV1TZXl4MmgJ7BSZVVVn24fr++j0+VJHVq46Pf39hLoYFetc6jZ+AoegaOoF/gKHoGjmpKPRMY6C1THX4xxK+OAABNkrubSXdeE6lHb+4j71ZmHc4o0uzFcdq4M73mHJvNruRDudoQf0zJh3Jls5G9AwAAuJpTpmacz/r167Vx40YZjUaNHDlSl156aUM8FgDQAgzoHqw5bXz01qok7T6Sr7dXJyvxUK76RATp4w0pyiuqqDk3wMdDd4zupkGRIS6sGAAAoGVzytSMb7/9Vi+//LIuvfRSPf/887XemzdvnpYuXVrr2NSpU/Xkk09e7GNxHkzNQHNEz+BsbDa7Vm8+rJU/HTzv1p4P39SbMAJnxfcZOIJ+gaPoGTiqKfVMg07N+P7775WWlqbBgwfXOp6YmKh33nlHdrtdbdu2VVhYmOx2u5YsWaKYmBhnPBoAAEmS0WjQDcPD9f8mDZDRcO5z31+7j2kaAAAALuKUIGLnzp2SpEsuuaTW8U8++USSdPXVV2vt2rX65ptvNHnyZNntdn344YfOeDQAALXY7HadL2PILarQ3qP5DVIPAAAAanNKEJGbmyuTyaTg4OBaxzdu3CiDwaDp06fLaDzxqAceeECStG3bNmc8GgCAWvJLKs5/kgPnAQAAwLmcEkQUFRXJ29u71rG8vDwdPnxYvr6+6tu3b83xkJAQeXp6Kjs72xmPBgCgFn9vjzqd593KrZ4rAQAAwJk4JYjw8vJSUVGRqqqqao5t3bpVktS/f//Tzndzc5PJZHLGowEAqKV7R38F+Jw/jHhnTbJikzPlhDWbAQAA4ACnBBGdO3eW3W7Xhg0bao6tWbNGBoNBgwYNqnVuWVmZioqKTpvGAQCAMxiNBt0xuts5z7F4mpVbVKk3ViZq/nvxOpRR2EDVAQAAwOyMm1x99dXatm2b/vKXv+jAgQPKzs7WV199JaPRqLFjx9Y6d+fOnbLb7erQoYMzHg0AwGkGRYbo4Zt6a/nafcor+m0tiEAfD00a3U19Ogfp69gj+mrzYe07VqC5S7bo0j5tdcvIzvKz1G1qBwAAAC6MU4KIO++8U1988YX27Nmjf/7znzXDXO+880517Nix1rnffvutDAbDaVt9AgDgTIMiQzSgW7BS0gpUZTfIzWBXl3Z+Mv66t+eNl0bosj5t9fGGFG1OzNTPO9MVtydLNwwP19WDO8rN7JRBgwAAAPgfTgkiPDw8tHz5cr3zzjvatm2bfHx8dOWVV+r666+vdV5lZaXi4uLUtm1bXXbZZc54NAAAZ2U0GtQjPFABAd7KyytRdbWt1vuBvq10/w29dNXADlq+dp8Ophfq4x9StGFbqiZe2U0Du7eWwWBwUfUAAADNk8HeRFbpys7O1saNG7Vr1y7t3LlTycnJqqioUHR0tJYtW3ZB91yzZo1++eUXJSYmKisrS/n5+XJzc1N4eLhGjhypKVOmKCAg4IzXxsTEaPv27TX1pKWlSZKWLl2qoUOHnvO5ZWVlWrZsmb799lsdOHBAFRUV8vf3V58+fXTbbbfpyiuvvKDP87+sVptyc0uccq/6YjYbz/oXBOBM6Bk4qq49Y7PbtTkxQx//kKL84kpJUlSYvyaN7q6OIZaGKheNAN9n4Aj6BY6iZ+CoptQzgYHeMpnOP6rUKSMiGsLq1as1b948p97zjTfe0O7du+Xu7q7g4GBFRkYqNzdXSUlJSkpK0ocffqhFixYpKirqtGsffvhhFRUVOfzM3Nxc3XnnnUpJSZEktW/fXv7+/kpNTdX69eu1fv163X333XrmmWcu+vMBAOrOaDBoeO+2Gtg9WF9tPqJvYo9o95F8zV4cq5H92mn8iM7y9XJ3dZkAAABNXr0EEcXFxUpKStLx48clSUFBQerZs6cslgv/jZLFYtHw4cPVp08f9enTR0lJSXr99dcvqs7JkycrIiJC/fv3l5vbb/vJ79mzRzNmzNDevXv1xBNPaPXq1add27VrV4WHh6tPnz7q3bu3pk+froKCgvM+85VXXlFKSooCAgL03//+VwMGDJAkWa1WLVu2TPPmzdPSpUt11VVXadiwYRf1+QAAjmvlbtbNIzprRN+2+vCHFG3ZnaUftqUpJjlL4y4N16hBHWSuQ9IPAACAM3NqEHFyscqffvpJNlvtISNGo1EjR47UH//4R0VGRjp87wkTJmjChAk1rzMzMy+63okTJ57xeGRkpF588UXdeuut2r9/v1JSUtSlS5da56xYsaLW67rOIf7+++8lSQ8++GBNCCFJJpNJU6dO1Q8//KBNmzZpw4YNBBEA4EKt/T310Pje2nMkT++v26cjmcVa8f1+rd+WpttHdVXfLkGsHwEAAHABnPYrnW+//VYTJ07Uhg0bZLVaZbfba/3ParVq/fr1mjhxor777jtnPbbedO7cuebrsrIyp923ouLENnJhYWFnfP/k8aqqKqc9EwBw4SLDAvTclCGaOjZKvl5uyswt1asf79A/P9yu1JzGvQ4PAABAY+SUIOLo0aOaMWOGKioq1K5dO82aNUvffvutduzYoR07dujbb7/VrFmz1L59e1VUVGjGjBk6evSoMx5db7Zu3SpJ8vLyUkREhNPu26NHD0lSQkLCae9ZrVZt27ZNktS3b1+nPRMAcHGMRoNG9GuneQ9corFDw2QyGrTrYK5mvR2r977bq+IywmMAAIC6ckoQ8fbbb6uyslL9+/fXF198oUmTJiksLEzu7u5yd3dXWFiYJk2apC+++EL9+/dXZWWlFi9e7IxHO5XNZlNmZqY+/fRTzZw5U5I0Y8YMeXt7O+0Zf/zjH+Xm5qZFixbprbfeUkZGhsrLy7Vnzx49/vjj2rNnj6Kjo3Xdddc57ZkAAOfw9DDr1iu76oXpQzWgW2vZ7Hat23pMMxds0rqtx2S1Ne6VrAEAABoDp6wRsWnTJhkMBs2ZM+ecf2n38vLSnDlzNG7cOG3cuNEZj3aKJUuWnLYjR9++fTV//nyNGDHCqc8aOnSoli5dqtdee01/+9vf9Le//a3mPYvFoieeeEJTp06VyWRyyvPM5sa9oNrJrV3qssULINEzcFx99Ez7YIseu62/Eg/m6r1v9+hYdone+26vftiWqslXd1fvzkFOexYaHt9n4Aj6BY6iZ+Co5tgzTgkiMjIy5O3tXadFKCMjI2WxWJSRkeGMRztFaGioBg4cKKvVqrS0NOXk5Cg5OVkrV65U//795evr69TnHTt2TDk5OZKkNm3aKCAgQEePHlVxcbFWrlypfv36aejQoRf9HKPRoIAA543mqE++vp6uLgFNDD0DR9VHz1wW4K1L+rXXNzGH9e6a3UrNLtFflycoumcb3XtjL7ULvvDdouB6fJ+BI+gXOIqegaOaU884JYgwm82qrq6u07l2u11VVVUym+tl59ALMnbsWI0dO7bm9e7duzV37lytWrVKKSkp+uSTT5w2QuGtt97S3/72N3Xq1EkfffRRzVoQJ7fvnD9/vu69916999576tev30U9y2azq7Cw1Bll1xuTyShfX08VFpbJamVIM86PnoGjGqJnLukRor7hAfr8pwNau+WYYpMytHV3pq6J7qhxl3WWV6vG8988nB/fZ+AI+gWOomfgqKbUM76+nnUaueGUn4w6deqk5ORk/fTTT7r88svPee5PP/2kioqK07bDbEyioqK0YMECjR49WsnJyVq9erVuvPHGi77v8ePH9dprr0mS5s+fX2tBypPbd+7du1effPKJ/vWvf+ntt9++6GdWVzfuRj3JarU1mVrRONAzcFR994yHm0m3jeqmEf3aacW6/dp54LjWbD6in3ek6+YRnXV533YyGtnusynh+wwcQb/AUfQMHNWcesYpk0xGjRolu92uZ599VikpKWc9b//+/XruuedkMBh01VVXOePR9cZisSg6OlqSlJiY6JR77tq1S+Xl5fLy8tKAAQPOeM5ll11Wcy4AoOlpG+Stxyb2059u7ae2QV4qKq3SO1/v0ZwlcdpzJM/V5QEAALicU0ZETJ06VR999JEyMjI0fvx4jRkzRpdccolCQ0MlnVhDYtOmTfrmm29UVVWlNm3aaMqUKc54dL06Od3EarU65X4lJXXfb76ystIpzwQAuEbfLkHqGR6g9fGpWvnzQR3NKtbLyxM0KDJYE6/sqmD/5jPPEwAAwBFOCSIsFoveeust/f73v1dqaqpWrVqlVatWnXae3W5Xhw4d9N///lcWS+NewCs/P1+xsbGSpB49ejjlnuHh4ZKk0tJSJSQkaODAgaed8/PPP0uSIiIinPJMAIDrmE1GXT2ko4b1CtXnPx/UDwmp2ronW9v3H9fvojvq2mGd5OnB+hEAAKBlcdr+H926ddMXX3yhxx9/XD169JDRaJTdbpfdbpfRaFSPHj00Y8YMrVy5Ut26dXPWY89r0qRJGjVqlJYsWVLreGxsrF5//XUdO3bstGsSExN17733qqioSKGhoRozZoxTaunRo4e6d+8uSXrqqae0c+fOmvesVquWLFmiTz/9VJI0btw4pzwTAOB6Pl7uuuuaSM25J1o9OgWo2mrT6k2H9fSbm7VxZ7psdrurSwQAAGgwBru9fn76qaqqUkFBgSTJz89Pbm5ukqSioiLdfffdMhgMNX/prov09HSNHz++5nVlZaVKS0tlNptrja647777NH369JrXo0aNUmpqqh555BE9+uijNcfXrl2rhx9+WJIUHByskJAQmUwmpaenKzs7W9KJbT0XLFhwxhERJ3fVOKmgoEB2u10Wi6XWjiAxMTG1rtu9e7emTp2qvLw8GQyGmu07jxw5ouLiYknSVVddpX/9618XvbOI1WpTbm7dp4O4gtlsVECAt/LySprNwiuoX/QMHNXYesZut2vbvhx98P1+ZeWXSZIi2vpo0lXd1bWDn4urg9T4egaNG/0CR9EzcFRT6pnAQO+G2zXjTNzc3NS6devTjldXVys5OVkGg2Mrh1utVuXn55/xfqceLy8vr9P9BgwYoJkzZyomJkb79+/XoUOHVFlZKV9fXw0dOlSjRo3ShAkTzjqFpKSk5Iz1nAwTziYqKkqrVq3SO++8ow0bNujo0aPKzs6Wr6+vLrvsMo0bN0433HCDw38+AICmwWAwaED3YPXuHKS1W4/qy42HdDC9SC+9u1VDe4bq1iu6KNC3lavLBAAAqDf1NiLibPLy8nTJJZfIYDAoOTm5IR/d4jAiAs0RPQNHNfaeKSip1KcbUvTzjnTZJbmbjRozNExjh3WSh5vJ1eW1SI29Z9C40C9wFD0DRzWlnqnriAinrREBAAAc5+ftrnuu7aHnpg5R9w5+qqy26YuNh/T0ws3anJShBv59AQAAQL0jiAAAoBHo1MZHT04eqAfH91aQbyvlFVVo4RdJmvduvA6mF7q6PAAAAKchiAAAoJEwGAwaEhWiF6cP1U0jOsvDzaT9qQWa+84Wvb0qSXlFFa4uEQAA4KIRRAAA0Mi4u5l0w/BwvXT/MF3Sq40kaeOuDD29cLNW/XJIVdVWF1cIAABw4QgiAABopAJ8PDT9hp565u5B6tLOVxVVVn364wE982aMtuzOYv0IAADQJBFEAADQyHVp56en7xqk6Tf0VICPh3IKyvX657v01+UJOpJZ5OryAAAAHGK+kIt69Ojh7DoAAMA5GAwGXdKrjQZ2C9aamMNaE3NEe47ma87iOF3er51uHtFZvt7uri4TAADgvC4oiGAoKAAAruHhbtL4yzvrsr5t9fEPKYpNztKP29MUtztTNwyP0OjBHWSuw/7dAAAArnJBQcQjjzzi7DoAAIADWvt56vfjemvUwHy9v26fDmcU6cP1+/XDtlTdPqqb+nUNksFgcHWZAAAApzHYGd7QbFmtNuXmlri6jHMym40KCPBWXl6Jqqttri4HTQA9A0e1hJ6x2e3auDNdn244oIKSSklSr/AA3X5VN7UPtri4uqanJfQMnId+gaPoGTiqKfVMYKC3THUYmcnYTQAAmjijwaDL+7bTS/cP07XDOslsMijxUJ5mLYrTu9/uUXFZlatLBAAAqEEQAQBAM+HpYdaEK7rohenDNLB7sGx2u76PT9XMBZv03ZajqrY27t+iAACAloEgAgCAZibE31OP3NxHf540QB2CLSopr9b7a/dp1qJY7Txw3NXlAQCAFo4gAgCAZqpHpwDNvmeI7v5dpCyebko/Xqp/frhd//fRdqUfb9xrCAEAgObrgnbNAAAATYPRaNAVA9orukeIvth4SOu2HtOOlONKPJirqwZ10I2XhsurlZurywQAAC0IIyIAAGgBvFq56faruun5e6PVt0uQrDa7vo07qqcWbNb6hFTZbGyiBQAAGgZBBAAALUjbIG/96dZ+enxiP7UN8lJxWZWWfbNHsxfHKvlQrqvLAwAALQBBBAAALVDvzkGaMy1ad4zuJu9WZh3LLtHfVmzTvz/dqaz8MleXBwAAmjHWiAAAoIUym4waPbijhvVqo5U/HdT6hFTF783WjpQcXT2ko66/JFyeHvyoAAAAnIsREQAAtHAWTzdNvqa75kwbol7hAaq22rVm8xHNXLhZP21Pk83O+hEAAMB5CCIAAIAkqX2wRY/f1l9/uKWvQgM8VVhSqcVrdmvuki3aezTf1eUBAIBmgiACAADUMBgM6t+ttebeN1QTr+wqTw+TDmcWaf578Xpj5S4dLyh3dYkAAKCJY+InAAA4jdlk1JihYRreu40+++mAftyWptjkLCXsy9GY6DBdO6yTPNxNri4TAAA0QYyIAAAAZ+Xr7a4pY6I0654hiuzor6pqm7785ZCefnOzNu3KYP0IAADgMIIIAABwXmGhPvp/dwzQQ+N7q7VfK+UVVejNVUmat2yrDqQVuro8AADQhBBEAACAOjEYDBocFaIXpw/VLSM7y8PNpJS0Qr2wdIve/DJJeUUVri4RAAA0AQQRAADAIW5mk667JFzzHhimS/u0kSRtSszQzIWb9OXGg6qssrq4QgAA0JgRRAAAgAvib/HQvdf11LNTBqtrez9VVtn02U8H9cybMYrbnSU760cAAIAzIIgAAAAXJaKtr2beOVD339hTAT4eOl5Yrv9+vksvvxevwxlFri4PAAA0MgQRAADgohkMBg3r2UYv3T9M4y6LkLvZqL3HCvT8kjgt/ipZBSWVri4RAAA0EgQRAADAaTzcTBp3WYReun+YhvYMlV3STzvSNXPBJq3ZfFhV1TZXlwgAAFyMIAIAADhdoG8rPXBjLz195yCFt/FReaVVH/2QomffilH83mzWjwAAoAUjiAAAAPWmawc//WXKYN17XQ/5WdyVlV+mf3+6U6+s2KZjWcWuLg8AALgAQQQAAKhXRoNBl/Zpq3n3D9N1l3SS2WRU8uE8zVocq2Xf7FFRKetHAADQkhBEAACABtHK3axbRnbRi9OHanBksOx2aX1CqmYu2Kxv446q2sr6EQAAtAQEEQAAoEEF+3vqoZv66Mk7BqhjiEWlFdVasW6fnns7VjtSjru6PAAAUM8IIgAAgEtEhgVo1tQhmjImUj5ebsrILdX/fbRd//xwu9KPl7i6PAAAUE/Mri4AAAC0XEajQSP7t9eQqFCt+uWQvttyVDsPHFfSoVxdObC9xl0WIe9Wbq4uEwAAOBEjIgAAgMt5tTJr4qiueuG+oerftbWsNrvWbjmmmQs26/v4Y7LaWD8CAIDmgiACAAA0GqGBXvrDhL564rb+at/aW8VlVXr3272avThOiYdyXV0eAABwAoIIAADQ6PSKCNTsaUM0+eru8m5lVmp2if6+Ypte+2SHMvNKXV0eAAC4CKwRAQAAGiWT0airBnXQ0J6h+uLng/o+PlUJ+3K0I+W4rh7SUTcMD5enBz/KAADQ1DAiAgAANGoWTzfdcXV3PX9vtHpHBMpqs+vrmCOauWCTftyeJpvN7uoSAQCAAwgiAABAk9Cutbcem9hPf5zQV6GBXiosrdKSNbv1/Dtx2nMkz9XlAQCAOmI8IwAAaDIMBoP6dW2tXhGB+n7rMa3ceEhHMov18vIEDY4M1sQru6q1v6erywQAAOdAEAEAAJocs8moa6LDNKx3G33+00Ft2JaqLXuytW3/cY0Z2lHXDuukVu78mAMAQGPE1AwAANBk+Xq56+7fRWr2PdHq0SlA1VabVv1yWE8v3KyNO9Nls7N+BAAAjQ1BBAAAaPI6hlg04/b+euTmPgr2b6X84kq9vTpZLy7dqv2pBa4uDwAAnIIgAgAANAsGg0EDuwfrhfuGacIVXeThbtLB9EK9tGyrFn6ZqNzCcleXCAAARBABAACaGTezUdcO66T59w/TZX3byiBpc2Kmnn5zs774+aAqqqyuLhEAgBaNIAIAADRLfhYPTbu2h56dOljdOvipssqmz38+qGfe3KyYpEzZWT8CAACXIIgAAADNWngbXz01eaB+P66Xgnw9lFtYoQVfJGree/E6mF7o6vIAAGhxCCIAAECzZzAYFN0jVC9OH6bxl0fI3c2o/ccK9MI7W7RodbIKiitcXSIAAC0GQQQAAGgx3N1MuvHSCL00fZgu6RUqu6Sfd6brqYWbtXrTIVVVs34EAAD1jSACAAC0OIG+rTT9hl565q5Bimjrq4pKqz7ZcEDPvBmjuN1ZrB8BAEA9IogAAAAtVpf2fnrm7kGafn1P+VvclVNQrtc+3qFn/vuLjmQWubo8AACaJbOrCwAAAHAlo8GgS3q30YDurfXV5iP6JvaIdqbkaNeBHI3o1043jegsXy93V5cJAECzwYgIAAAASa3czbp5RGe9/PtLdFm/drLbpQ3b0jRzwWZ9E3tE1Vabq0sEAKBZIIgAAAA4RWt/Tz159xA9c/cghYVaVFZRrQ++369n347Vtv05rB8BAMBFIogAAAA4g8iwAD03ZYimjo2Sr5ebMnNL9a+Pd+gfH25Xak6Jq8sDAKDJIogAAAA4C6PRoBH92mneA5do7NAwmU0GJR7M1ay3Y/Xed3tVXFbl6hIBAGhyCCIAAADOw9PDrFuv7KoX7huqAd1ay2a3a93WY5q5YJPWbT0mq431IwAAqCuCCAAAgDoKCfDSo7f01Yzb+6t9sLdKyqv13nd7NWtRnHYdPO7q8gAAaBIIIgAAABzUMzxQs+8Zoruu6S6Lp5vSckr0jw+269WPtisjt9TV5QEA0KiZXV0AAABAU2QyGnXlwA6K7hmqL34+pO/jj2l7ynHtOpir0YM76Ibh4fJq5ebqMgEAaHQYEQEAAHARvFu5adLobnr+3mj17RIkq82ub2KPaubCzfphW6psNrb7BADgVAQRAAAATtA2yFt/urWf/nRrP7UN8lJRaZWWfr1Hc5bEaffhPFeXBwBAo8HUDAAAACfq2yVIPcMDtD4+VSt/PqijWcX66/sJGtQ9WLeO6qoQf09XlwgAgEsRRAAAADiZ2WTU1UM6alivUH3+80H9kJCqrXuztT0lR7+LDtO1wzrJ04MfwwAALVOT+S9gdna2Nm7cqF27dmnnzp1KTk5WRUWFoqOjtWzZsgu655o1a/TLL78oMTFRWVlZys/Pl5ubm8LDwzVy5EhNmTJFAQEBZ7w2JiZG27dvr6knLS1NkrR06VINHTq0Ts+Pj4/X8uXLtWXLFh0/flze3t5q3769oqOj9fvf/15+fn4X9LkAAEDj4OPlrruuidSVA9rr/bX7lHw4T6s3HdbPO9J1y8guGt6njYwGg6vLBACgQTWZIGL16tWaN2+eU+/5xhtvaPfu3XJ3d1dwcLAiIyOVm5urpKQkJSUl6cMPP9SiRYsUFRV12rUPP/ywioqKLvjZr7zyit566y3Z7Xa1bt1akZGRKioq0v79+7Vr1y7ddNNNBBEAADQTHYItmnF7f23bl6MPvt+vrPwyLfoqWd/HH9Ok0d3UrYO/q0sEAKDBNJkgwmKxaPjw4erTp4/69OmjpKQkvf766xd1z8mTJysiIkL9+/eXm9tv22vt2bNHM2bM0N69e/XEE09o9erVp13btWtXhYeHq0+fPurdu7emT5+ugoKCOj33v//9r95880116tRJL7zwgqKjo2veq6qqUlxcnIKDgy/qswEAgMbFYDBoQPdg9e4cpLVbj+rLjYd0KKNI896NV3SPEN16RVcF+bVydZkAANS7JhNETJgwQRMmTKh5nZmZedH3nDhx4hmPR0ZG6sUXX9Stt96q/fv3KyUlRV26dKl1zooVK2q9NtRxWOWBAwf0n//8Rz4+Plq6dKnatGlT6303NzcNHz7cgU8BAACaEjezUWOHdtLw3m312Y8p+ml7umKTs7RtX47GDA3T2GGd5OFmcnWZAADUG7bvPIvOnTvXfF1WVua0+7777ruqqqrSrbfeeloIAQAAWg4/b3dNHdtDz00dou4d/VVZbdMXGw/p6YWbtTkxQ3a73dUlAgBQL5rMiIiGtnXrVkmSl5eXIiIinHbfdevWSZIuueQSpaWlacWKFUpKSpLBYFDXrl118803q1u3bk57HgAAaNw6tfHRk3cM0NY92frg+/06XliuhV8maV38Md0xursi2vq6ukQAAJyKIOIUNputZneOV155RZI0Y8YMeXt7O+X+OTk5ysjIkCQdOXJEjz32mIqLi2ve//HHH7VkyRLNmDFD9957r1OeCQAAGj+DwaDBUSHq2yVI38Qd1VebDisltVBz39mi4b3b6JaRXRTg4+HqMgEAcAqCCElLliw5bUeOvn37av78+RoxYoTTnpOdnV3z9fz589W+fXv9+9//1qBBg5Sbm6u33npLy5Yt01//+ld17txZV1555UU/02xu3LNvTCZjrX8C50PPwFH0DBzlyp4xm426aURnXTGgvT78fr827kzXL7sytHVPtm64NFxjhoXJ3cz6EY0J32PgKHoGjmqOPUMQISk0NFQDBw6U1WpVWlqacnJylJycrJUrV6p///7y9XXOkMiSkpKar202mxYsWKDw8HBJUps2bfSXv/xFqamp+v777/Xaa69ddBBhNBoUEOCc0Rz1zdfX09UloImhZ+AoegaOcmXPBAR466mp0dp7JE8LP9+pPYfz9PEPKfpxR7qmXd9Lw/u2rfNC2WgYfI+Bo+gZOKo59QxBhKSxY8dq7NixNa93796tuXPnatWqVUpJSdEnn3wik+nif/vg4fHbkMorrriiJoQ41bRp0/T9998rMTFRubm5CgwMvODn2Wx2FRaWXvD1DcFkMsrX11OFhWWyWm2uLgdNAD0DR9EzcFRj6plgH3c9fedAbdqVoQ++36+s3FLNXxqnyDB/3XlNpDq18XFpfWhc/YKmgZ6Bo5pSz/j6etZp5AZBxBlERUVpwYIFGj16tJKTk7V69WrdeOONF31fPz+/mq//dzvQMx0/duzYRQURklRd3bgb9SSr1dZkakXjQM/AUfQMHNWYeia6R6j6dWmtNTGHtSbmiPYcyddzb8Xo8n7tdPOIzvL1dnd1iS1eY+oXNA30DBzVnHqm+UwycTKLxaLo6GhJUmJiolPu2b59+5pREW5ubmc8x939tx8k2LYLAACc5OFu0vjLO+ul6cMU3SNEdkk/bk/TzIWb9HXMEVU38t+SAQBwEkHEOVRXV0uSrFarU+5nMpnUr18/SdLRo0fPeM6px9u0aeOU5wIAgOYjyK+Vfj+ut56aPFCd2viorMKqD9fv11/eilHCvmx+kQEAaPQIIs4iPz9fsbGxkqQePXo47b4n16L4/vvvVVBQcNr7n3zyiSQpIiJCoaGhTnsuAABoXrp39NezUwbrnmuj5Oftrqy8Mr32yU79/YNtOpZdfP4bAADgIs0+iJg0aZJGjRqlJUuW1DoeGxur119/XceOHTvtmsTERN17770qKipSaGioxowZ47R6JkyYoA4dOqi4uFgzZ85UUVFRzXtr1qzRihUrJEm///3vnfZMAADQPBkNBl3et51eun+Yrh3WSWaTQUmH8jRrUayWfbtHxWVVri4RAIDTNJnFKtPT0zV+/Pia15WVlZKk+Ph4DR06tOb4fffdp+nTp9e8zszMVGpqaq2/8EtSYWGhXn31Vb366qsKDg5WSEiITCaT0tPTlZ2dLenEtp4LFiyQt/fpW2Ce3FXjpJOjGx566CGZzb/9scbExNS6zt3dXf/5z380ZcoUrVu3Tpdffrm6dOmivLw8paamSpLuvPPOWp8VAADgXDw9zJpwRReN6N9OH32/X1v3Zmt9fKpikzJ142URunJAe5mb0f7zAICmrckEEVarVfn5+acdr66urnW8vLy8TvcbMGCAZs6cqZiYGO3fv1+HDh1SZWWlfH19NXToUI0aNUoTJkyQxWI54/UlJSVnrKe4+PxDIaOiorRq1SotWLBA69ev1549e+Tp6anhw4dr8uTJGj16dJ0+AwAAwKlC/D318M19lHw4T++v3adj2cV6f+0+/ZCQqtuv6qY+nYNcXSIAADLYWdGo2bJabcrNLXF1GedkNhsVEOCtvLySZrMVDeoXPQNH0TNwVHPpGZvNrh+3p+nTHw/UTNHo2yVIt43qqrZBp4/2xIVpLv2ChkPPwFFNqWcCA71lqsMIvCYzIgIAAAB1ZzQadMWA9oruEaIvNh7Suq3HtCPluBIP5uqqQR1046Xh8mp15u3EAQCoT0wWBAAAaMa8Wrnp9qu6ae59Q9WvS5CsNru+jTuqpxZs1vqEVFltjfu3awCA5ocgAgAAoAVoE+ilP97aT49P7Ke2QV4qLqvSsm/2aM7iOCUfynV1eQCAFoQgAgAAoAXp3TlIc6ZF647R3eTdyqxj2SX624pteu2THcrKK3V1eQCAFoA1IgAAAFoYs8mo0YM7alivNlr500GtT0hVwr4c7TxwXFcP7qjrh4fL04MfEwEA9YMREQAAAC2UxdNNk6/prjnThqhXRKCqrXatiTmimQs366ftabKxuRoAoB4QRAAAALRw7YMtenxiP/1hQl+FBniqsKRSi9fs1twlW7T3aL6rywMANDOMuQMAAIAMBoP6d22t3hGBWrvlmL785aAOZxZp/nvxGhIVoluv7KLWfp6uLhMA0AwQRAAAAKCG2WTUmKFhGt67jT776YB+3JamuN1Z2rY/R7+LDtN1wzrJw93k6jIBAE0YUzMAAABwGl9vd00ZE6VZ9wxRVJi/qqptWvXLIc1cuEmbdmWwfgQA4IIRRAAAAOCswkJ99OdJA/TQ+N5q7ddK+cWVenNVkl5atlUpaQWuLg8A0AQRRAAAAOCcDAaDBkeF6MXpQ3XLyM7ycDPpQFqhXly6VW9+mai8ogpXlwgAaEIIIgAAAFAnbmaTrrskXPMeGKZL+7SRJG1KzNTMhZv0xcaDqqyyurhCAEBTQBABAAAAh/hbPHTvdT317JTB6treT5VVNn3+00E982aMYpMzZWf9CADAORBEAAAA4IJEtPXVzDsH6oEbeynAx0PHC8v1xspEvfxevA5nFLm6PABAI0UQAQAAgAtmMBg0tGeoXrp/mMZdFiF3s1F7jxXo+SVxWvRVsgqKWT8CAFAbQQQAAAAumoebSeMui9BL9w/TsJ6hskv6eUe6Zi7crK82H1ZVtc3VJQIAGgmCCAAAADhNoG8r3X9jLz195yCFt/FReaVVH/+Qor+8tVnxe7NZPwIAQBABAAAA5+vawU9/mTJY917XQ34Wd2Xnl+vfn+7UKyu26VhWsavLAwC4EEEEAAAA6oXRYNClfdpq3v3DdN0lnWQ2GZV8OE+zFsdq6Td7VFha6eoSAQAuQBABAACAetXK3axbRnbRi9OHanBksOx26YeEVM1csFnfxh1VtZX1IwCgJSGIAAAAQIMI9vfUQzf10ZN3DFBYiEVlFdVasW6fnns7VjtSclxdHgCggRBEAAAAoEFFhgXoualDNGVMpHy83JSRW6r/+2iH/vHhNqXllLi6PABAPSOIAAAAQIMzGg0a2b+95t1/icZEh8lkNGjXgVw993asln+3VyXlVa4uEQBQTwgiAAAA4DJercyaOKqrXrhvqPp3bS2b3a61W4/pqTc2ad3WY7LaWD8CAJobgggAAAC4XGigl/4woa+euK2/2rf2Vkl5td77bq9mL4pT4qFcV5cHAHAigggAAAA0Gr0iAjV72hBNvrq7vFuZlZpTor+v2KZ/fbxDmXmlri4PAOAEZlcXAAAAAJzKZDTqqkEdNLRnqL74+aC+j0/Vtv052nnguK4e3FHXDw+XVyt+jAWApooREQAAAGiULJ5uuuPq7nr+3mj17hwoq82ur2OP6OmFm/Tj9jTZbHZXlwgAuAAEEQAAAGjU2rX21uMT++tPt/ZVm0AvFZZWacma3Xp+SZz2HMlzdXkAAAcxpg0AAABNQt8urdUzPFDfbz2mlRsP6UhWsV5enqDBkcGaeGVXtfb3dHWJAIA6IIgAAABAk2E2GXVNdJiG9W6jz386qA3bUrVlT7a27T+u30V31HWXdFIrd37EBYDGjKkZAAAAaHJ8vdx19+8iNfueaPXoFKBqq02rNx3WzIWbtXFnumx21o8AgMaKIAIAAABNVscQi2bc3l+P3NxHwf6tVFBcqbdXJ+vFpVu0P7XA1eUBAM6AIAIAAABNmsFg0MDuwXrhvmG69You8nA36WB6kV5atlULv0hUbmG5q0sEAJyCCXQAAABoFtzMRo0d1knDe7fRJz8e0MYd6dqclKn4vdkaO6yTxgwNk4ebydVlAkCLx4gIAAAANCt+Fg9Nu7aHnp06WN06+Kmy2qaVPx/UM29uVkxSpuysHwEALkUQAQAAgGYpvI2vnpo8UL8f10tBvh7KLazQgi8SNe+9eB1ML3R1eQDQYhFEAAAAoNkyGAyK7hGqF6cP0/jLI+TuZtT+YwWa+84Wvb06SfnFFa4uEQBaHIIIAAAANHvubibdeGmEXpo+TJf0CpUkbdyZoZkLN2v1pkOqqra6uEIAaDkIIgAAANBiBPq20vQbeumZuwapcztfVVRa9cmGA3rmzRht3ZPF+hEA0AAIIgAAANDidGnvp6fvGqTp1/eUv8VdOQXl+s9nu/S39xN0JLPI1eUBQLNGEAEAAIAWyWgw6JLebTTv/kt0w/BwuZmN2n0kX3OWxOmdr3ersKTS1SUCQLNEEAEAAIAWzcPdpJtGdNaL04dqSFSI7HZpw7Y0zVy4SV/HHFG11ebqEgGgWSGIAAAAACS19vPUg+N766nJA9Up1EdlFVZ9uH6/nn0rRtv257B+BAA4CUEEAAAAcIruHf317JTBumdslHy93JSZV6Z/fbxD//hwu1JzSmrOs9nsSj6Uqw3xx5R8KFc2G0EFANSF2dUFAAAAAI2N0WjQ5f3aaXBUiFb9ckjfbTmqxIO5mvV2rK4c0F7hbX306Y8HlFdUUXNNgI+H7hjdTYMiQ1xYOQA0foyIAAAAAM7C08OsW6/sqhfuG6oB3VrLZrdrXfwxvb06uVYIIUl5RRX6z2e7tHVPlouqBYCmgSACAAAAOI+QAC89ektfPTGxn0xGwznPfX/tPqZpAMA5EEQAAAAAdWQyGWU9T8iQW1Sh9Qmpqqi0NlBVANC0sEYEAAAAUEf5JRXnP0nSe9/t1fLv9io00EthoRaFhfqc+GeIj3y93eu5SgBo3AgiAAAAgDry9/ao03nercwqKa9WRm6pMnJLFZv827oR/hb3WsFEWKhFrf09ZTSce8oHADQXBBEAAABAHXXv6K8AH4/TFqo8VaCPh/764HAVlVbqSFaxjmQW6UhmsY5kFSsrt1T5xZXKLz6uHSnHa65p5W5SWIhFHU8JKNq19pabmZnUAJofgggAAACgjoxGg+4Y3U3/+WzXWc+ZNLqbjEaD/Cwe6mPxUJ/OQTXvlVVUKzW7RIczi3Q0q0iHM4uVml2s8kqr9h4r0N5jBTXnmowGtWvtrbCQ36Z2dAzxkVcrfoQH0LTxXQwAAABwwKDIED18U28tX7uv1siIQB8PTRrdTYMiQ856raeHWV07+KlrB7+aY9VWmzKOl+pI1q8jJ34dQVFaUa2jWcU6mlWsjbsyas5v7ddKnUJ91PHk2hMhFgX4eMjA1A4ATQRBBAAAAOCgQZEhGtAtWClpBaqyG+RmsKtLOz8Zz7O155mYTUZ1CLGoQ4hFw3ufOGa323W8sLxWMHE0q0jHCyuUU1CunIJybd2bXXMPi6fbb4ti/jqCok2g1wXVAwD1jSACAAAAuABGo0E9wgMVEOCtvLwSVVfbnHZvg8Gg1n6eau3nqYHdg2uOF5dV6WjmiSkdR38dQZF+vFTFZVVKOpSnpEN5Nee6m08EHCeDiY6hFnUItsjDzeS0OgHgQhBEAAAAAE2ExdNNPcID1SM8sOZYZZVVqTklpyyKWaSjWcWqrLLpQFqhDqQV1pxrMEhtAr1Om9rh48WWogAaDkEEAAAA0IS5u5kU0dZXEW19a47ZbHZl5pXqaFbxiYUxf53iUVhapfTjpUo/XqrNSZk15wf4eNRaFDMs1Eet/Vqx7gSAekEQAQAAADQzRqNBbYO81TbIW9E9QiWdWHeioKTyt5ETmUUnthTNK1NeUYXyiiq0/ZQtRT09zOoYYqnZTjQs1KJ2rb1lNrGlKICLQxABAAAAtAAGg0H+Fg/5WzzUt0vrmuNlv+7OcTKYOJJZpNTsEpVVVGvv0XztPZpfc67ZdHJL0d9GTnQMscjTg79WAKg7vmMAAAAALZinh1ndO/qre0f/mmPVVpvSckpqT+3IKlZZRfWvoymKpZ2/3SPE31NhoRZ1DPVRp1CLOob4yN/iztQOAGdEEAEAAACgFrPJ+Ot6ET66tE9bSSemduQUlJ+yneiJkCKvqEJZ+WXKyi/Tlj2/bSnq6+WmjifXnPh1BEVoAFuKAiCIAAAAAFAHBoNBwf6eCvb31KDIkJrjRaWVOpJVXLMg5pGsYqUfL1FhaZUSD+Yq8WBuzbnubsYT606cMrWjfWtvubOlKNCiEEQAAAAAuGA+Xu7qFR6oXqdsKVpRZVVqdkmtdSeO/bqlaEpqoVJSf9tS1GgwqG2Q14ntRE8JKCyebq74OAAaAEEEAAAAAKfycDOpcztfdW5Xe0vRjNxSHcn6dWpHZpEOZxaruKxKqTklSs0p0ebE37YUDfL1UMdTgomwEIuC2FIUaBYIIgAAAADUO6PxxI4b7Vp7a1jPE8fsdrvyiyt/XRDz121Fs4qUnV+u44UVOl5YoW37c2ru4eVhrrVbR6dQH7UJ8mJLUaCJIYgAAAAA4BIGg0EBPh4K8PFQ/66/bSlaWl6to1m/BRNHM4uVmlOi0opq7T6Sr91H8mvONZuMah/srbCQX0dOhFrUMcSiVu78VQdorPh/JwAAAIBGxauVWZFhAYoMC6g5VlVtU/rxkt+2E/11/YnySqsOZxTpcEaRpHRJkkFSSIBnTTBxcmqHn8XDNR8IQC0EEQAAAAAaPTfzb1uKnmSz25WTX1YzcuLIrwFFfnGlMvPKlJlXprjdWTXn+3m7q2PoiSkdJ6d2BAd4ysi6E0CDajJBRHZ2tjZu3Khdu3Zp586dSk5OVkVFhaKjo7Vs2bILuueaNWv0yy+/KDExUVlZWcrPz5ebm5vCw8M1cuRITZkyRQEBAWe8NiYmRtu3b6+pJy0tTZK0dOlSDR061KE6du/erVtuuUXV1dWSpD179lzQ5wEAAABaEqPBoJAAL4UEeGlw1G9bihaWVNZM6TicWaSjWcXKOF6qgpJKFRzI1a4Dv20p6uFmOrGlaOhvUzvat/aWm5ktRYH60mSCiNWrV2vevHlOvecbb7yh3bt3y93dXcHBwYqMjFRubq6SkpKUlJSkDz/8UIsWLVJUVNRp1z788MMqKiq66BqsVqv+8pe/1IQQAAAAAC6Or7e7ekcEqXdEUM2xikqrjmUX195SNLtEFVVW7U8t0P7UgppzTcZftxQN8VGnUIs6/hpQeLdiS1HAGZpMEGGxWDR8+HD16dNHffr0UVJSkl5//fWLuufkyZMVERGh/v37y83tt28qe/bs0YwZM7R371498cQTWr169WnXdu3aVeHh4erTp4969+6t6dOnq6Cg4LTzzmfp0qXauXOnrrrqKq1bt+6iPg8AAACAM/NwN6lLez91ae9Xc8xqsynjeKmOZBXXjJ44klmkkvJqHcsu0bHsEm1K/O0eQb6tao2cCAvxUaCvB1uKAg5qMkHEhAkTNGHChJrXmZmZ5zi7biZOnHjG45GRkXrxxRd16623av/+/UpJSVGXLl1qnbNixYpary/km8+xY8f0r3/9S7169dJdd91FEAEAAAA0IJPRqPbBFrUPtuiSXieO2e125RVV1Kw3cXL0RE5BuY4Xnvhfwr7fthT1bmWuFUyEhVrUJshLJiNbigJn02SCiIbWuXPnmq/Lysrq5RmzZs1SRUWFnn/+eZWUlNTLMwAAAADUncFgUKBvKwX6tlL/br9tKVpSXnVit45fg4kjmcVKP16ikvJqJR/OU/LhvJpz3cxGdQj2rjW1o2OwRR7urDsBSAQRZ7V161ZJkpeXlyIiIpx+/88//1w///yz7r77bvXu3VsxMTFOfwYAAAAA5/Bu5aaoTgGK6nTqlqJWpeWU1mwpejjrxMKYFZVWHUwv0sH039aUM0gKDfRSpzY+iooIUoivh9q39pavt7sLPg3gWgQRp7DZbDW7c7zyyiuSpBkzZsjb29upz8nNzdW8efPUpk0b/fGPf3TqvQEAAAA0DDezSZ3a+KhTm9pbimbnldUaOXEks0gFJZXKyC1VRm6pYpJ+m2bub3FX2CnbiXYMtSjYny1F0bwRREhasmTJaTty9O3bV/Pnz9eIESOc/rwXX3xR+fn5+ve//y2LxeL0+5/KbG7cc9NMJmOtfwLnQ8/AUfQMHEXPwBH0C86kfYhF7UMsuqR3m5pj+cUVOpJZpKNZJUo7Xqr9R/OUcbxU+cWVyi8+rh0px2vObeVuUljoiYAjLNSiTm181L61RW6N/Gd71I/m+H2GIEJSaGioBg4cKKvVqrS0NOXk5Cg5OVkrV65U//795evr67RnbdiwQatWrdKoUaN09dVXO+2+Z2I0GhQQ4NzRHPXF19fT1SWgiaFn4Ch6Bo6iZ+AI+gXnExDgrYiOgbWOlVVU61BaoQ6kFehAaoEOpBXocHqhyiut2ns0X3uP5tecazYZ1DHURxHtTuz8EdHeT53b+cnbky1FW4rm9H2GIELS2LFjNXbs2JrXu3fv1ty5c7Vq1SqlpKTok08+kcl08QvLlJSUaPbs2fLy8tKzzz570fc7H5vNrsLC0np/zsUwmYzy9fVUYWGZrFabq8tBE0DPwFH0DBxFz8AR9Asc9b8908bfQ238QzS8Z4gkqdp6YkvRwxlFOpxZpMMZv20pejCtUAfTCvX9lqM19wv291SnNpaaERSdQn0U4MOWos1JU/o+4+vrWaeRGwQRZxAVFaUFCxZo9OjRSk5O1urVq3XjjTde9H3/9a9/KS0tTU899ZTatWvnhErPr7q6cTfqSVarrcnUisaBnoGj6Bk4ip6BI+gXOOpcPdMm0EttAr00tGeopBNbih4vLD+xIGbmiQUxj2QW6XhhhbLzy5SdX6Ytu7Nrrrd4up3YTjTUR2EhJ/7ZJtBLRiPhRFPWnL7PEESchcViUXR0tL755hslJiY6JYhISkqSJC1cuFBvvfVWrfeqqqpqvr700kslSc8884yuvfbai34uAAAAgKbLYDCotZ+nWvt5akD34JrjxWVVOppZ9NvCmFnFSs8pVXFZlZIO5Snp0G9birqbjeoQYlFYyIntRMNCLeoQbJGHG1uKouERRJxDdXW1JMlqtTr1vrm5ued8PycnR5JUXl7u1OcCAAAAaD4snm7qER6oHuG/rT1RWWVVak5JTTBx5NcRFJVVNh1IK9SBtMKacw2GE6Mvwn4NJsJCTvzTx4stRVG/CCLOIj8/X7GxsZKkHj16OOWey5YtO+t7MTExuvvuuyVJe/bsccrzAAAAALQs7m4mRbT1VUTb3xbct9nsyswr1dGsX6d2/LqlaGFpldKPlyr9eO0tRQN8PGpGTnQKPfHPYL9WrDsBp2n2QcSkSZOUmZmpu+++W1OnTq05Hhsbqy1btujGG29Uhw4dal2TmJio5557TkVFRQoNDdWYMWMauGoAAAAAcA6j0aC2Qd5qG+St6B6hNcdPbClaXGv0RFZemfKKKpRXVKHtp2wp6ulhUseQ2iMn2rX2lrkZbSmJhtNkgoj09HSNHz++5nVlZaUkKT4+XkOHDq05ft9992n69Ok1rzMzM5WamqqioqJa9yssLNSrr76qV199VcHBwQoJCZHJZFJ6erqys08s9BIaGqoFCxbI2/v0LTBP7qpxUkFBgSTpoYcektn82x9rTEzMRXxqAAAAAKgf/hYP+Vs81LdLUM2xsopqHc0qrjV6IjWnWGUVZ95StF1r75pgIizURx1DLPL0aDJ/zYSLNJkOsVqtys/PP+14dXV1reN1XVdhwIABmjlzpmJiYrR//34dOnRIlZWV8vX11dChQzVq1ChNmDBBFovljNeXlJScsZ7i4uI6PR8AAAAAGhtPD7O6d/RX947+NceqrTalHy89MXLilBEUZRXVv74ulnb+do8Qf0+FhZ4ytSPER/4Wd6Z2oIbBbrfbXV0E6ofValNubomryzgns9mogABv5eWVNJutaFC/6Bk4ip6Bo+gZOIJ+gaOaS8/Y7XblFJTXBBMnR1DkFVWc8XxfL7ea3TpOjqAIDWBL0bpoSj0TGOgtUx2m6zSZEREAAAAAgMbBYDAo2N9Twf6eGhT525aiRaWVOpJVfGJBzKwTIyjSj5eosLRKiQdzlXjwtx0E3d2M6hj865SOUIs6hfqofWtvubOlaLNHEAEAAAAAcAofL3f1Cg9Ur//ZUvRYdklNMHH0lC1FU9IKlXLKlqJGg0Ftg7zU8ZSRE2GhPrJ4urni46CeEEQAAAAAAOqNu5tJndv5qnO707cUPXU70cOZxSouq1JqTolSc0q0OfG3LUUDfT1qBRNhIRYFsaVok0UQAQD4/+3de1BV19nH8d8BDig3BQUU4i2+crwVo6bRNDZOycWkSY1JjdbYsTZGM1GjnehU06pNay02tR2rJtE6RaiJ0SQzxo6mShNjxpIIKSPeAr7ghQiioIhyUQ5w9vsHcoQXldu5cA7fz4wTzt5rP6yNT1ZWHvZeCwAAwKUabik6dmjdMcMwVFpuvbko5q0tRYtLb6jkWpVKrlUpM/eSPUZggF+j3Tr6RYWoV49AthT1ABQiAAAAAABuZzKZFBYSoLCQAI34n57245U3anSu6FZhom5L0QpVVtUo+9tSZX9bam/r5+ujmIgg9Y28+eREVLDuiWBL0Y6Gvw0AAAAAQIcV2MVPlr5hsvQNsx+rqbXp/KWKRtuJnisq0/WqWuVdKFPehTJJhZIkk6TIsK6NthPtFxWsbsEB7rkhUIgAAAAAAHgWP1+fm088hEjqLUmy1W8peqHB0xNF5bpSVqWLV67r4pXr+m92kT1GaJB/o+1E+0aFKDKsq3xYd8LpKEQAAAAAADyej8mkyO5dFdm9q+4fHGk/fq3CqnNF5Y3WnbhwuVLXKqw6frpEx0/f2lI0wOyrPpHBtxbFjApWTM8gmf3YUtSRKEQAAAAAALxWaJC/hg0I17ABt7YUrbLWKr+43F6Y+PZiufKLy1VVXavcgqvKLbhqb+vrc3NL0ZuvdPS5WaAI6sKWom1FIQIAAAAA0KkE+PtqYEw3DYzpZj9Wa7PpQsl1+4KYeTd376i4UaP84grlF1foqxO3YvQI7dLoyYm+kSEKDw1gS9EWoBABAAAAAOj0fH18FNMzSDE9g/TgsLpjhmHoSllV3aKYRWX2xTEvXb2hy9fq/hzOubWlaFAXv0aFiT5RwerdI1C+Pm3bUtRmM5R1tkTVZ67IbDI0MLqbfHw8v9BBIQIAAAAAgNswmUwKD+2i8NAuum9Qwy1Fq3WuqFx5F8t17mKZ8i6Wq/ByhSpu1Cgr74qy8q7Y2/r5+uieiKBbBYqoEPWJCFaA/93Xncg4WaRtn+boSlmV/VhYSIBeeHSQRlsi73Jlx0chAgAAAACAVgjsYm6ypWh1Tf2WomW3nqAoKleVtVZnL5Tp7IUye1uTpKjwwFuvdkTW/TM0yF9SXRHirZ3Hm3zfK2VVemvncc17drhHFyMoRAAAAAAA0E5mPx/16xWifr1C7MdshqHi0uv2VzrqCxRXy626UFKpCyWVSs+6taVot2B/9Y0MVk7+1dt9C7v3P83RyEERHvuaBoUIAAAAAACcwMdkUlRYoKLCAvXdBluKXq2w3nylo8z+ikdRSaWullt1rLzkLhHrlJRV6X/PlWpwv7Bm23ZEFCIAAAAAAHChbkH+6nZvDw2/t4f92A1r3e4cX2QWKPXYhWZjlFZUNdumo2rb0p0AAAAAAMBhuvj76X9iuumh4b1b1L57UICTe+Q8FCIAAAAAAOggYvt0V1jI3YsM4SEBiu3T3TUdcgIKEQAAAAAAdBA+Pia98Oigu7aZ9uggj12oUqIQAQAAAABAhzLaEql5zw5v8mREeEiAx2/dKbFYJQAAAAAAHc5oS6RGDorQqfNXVW2YZDYZGhjdzaOfhKhHIQIAAAAAgA7Ix8ekIf3DFRYWpCtXKlRTY3N3lxyCVzMAAAAAAIDLUIgAAAAAAAAuQyECAAAAAAC4DIUIAAAAAADgMhQiAAAAAACAy1CIAAAAAAAALkMhAgAAAAAAuAyFCAAAAAAA4DIUIgAAAAAAgMtQiAAAAAAAAC5DIQIAAAAAALgMhQgAAAAAAOAyFCIAAAAAAIDLmAzDMNzdCTiHYRiy2Tr+X6+vr49qa23u7gY8CDmD1iJn0FrkDFqDfEFrkTNoLU/JGR8fk0wmU7PtKEQAAAAAAACX4dUMAAAAAADgMhQiAAAAAACAy1CIAAAAAAAALkMhAgAAAAAAuAyFCAAAAAAA4DIUIgAAAAAAgMtQiAAAAAAAAC5DIQIAAAAAALgMhQgAAAAAAOAyFCIAAAAAAIDLUIgAAAAAAAAuQyECAAAAAAC4DIUIAAAAAADgMhQiAAAAAACAy/i5uwPwHocOHdKWLVt05MgRVVZWKjo6Wk888YTmzJmjwMDANsXct2+f3n33XWVnZ6u6ulr9+vXTxIkTNWPGDJnNZgffAVzNkTmzdOlS7dy5865tNm/erIcffrg9XYabFBcXKzU1VcePH9exY8eUlZWlqqoqPfDAA9q6dWu7Yjtj7IL7OSNn1q9frw0bNty1zRtvvKFp06a1KT7cxzAMHT58WPv371dGRoZOnz6t8vJyhYSEaOjQoZo0aZJ+9KMfyWQytSk+8xnv46ycYT7j3f71r3/pyy+/1IkTJ1RUVKTS0lKZzWb1799f48eP189+9jOFhYW1KbanjTMUIuAQW7du1apVq2QYhnr16qXevXsrNzdX77zzjlJSUrRt2zZ17969VTH/+Mc/KjExUZLUt29fde3aVTk5OXrzzTf1+eefKzExUf7+/k64G7iCM3JGknr37q3evXvf9ly3bt3a2Wu4y549e5SQkODwuM7KQ7ifs3JGknr06KF+/frd9lxERIRTviec69ChQ5o5c6b9c58+fRQTE6OCggKlpqYqNTVVe/bs0fr161s992A+452cmTMS8xlvtXHjRmVnZ8vf318RERGyWCwqKSnRN998o2+++UYffPCBEhMTNXjw4FbF9chxxgDa6dixY8bgwYMNi8VibN++3bDZbIZhGMaFCxeMZ5991oiNjTXmz5/fqpgpKSlGbGysMXz4cOPTTz+1H8/NzTXi4+ON2NhYIyEhwaH3AddxRs4sWbLEiI2NNdatW+eMLsPNPvzwQ2PmzJnGn//8ZyMlJcVYu3atERsba/z0pz9tc0xn5CE6DmfkzLp164zY2FhjyZIlDuwpOoLU1FQjPj7eSE5ONi5dutTo3M6dO43hw4cbsbGxxptvvtmquMxnvJezcob5jHfbsWOHkZ6eblit1kbHs7OzjaefftqIjY01fvjDH7YqpqeOM6wRgXZ7++23ZbPZ9Mwzz2jq1Kn2R9CioqL0l7/8RT4+PkpJSVF2dnaLY9Y/+jp79mw98sgj9uMDBw7U73//e0nSe++9p5KSEgfeCVzFGTkD7zZ58mRt2bJFr732mh577DH16NGj3THJQ+/mjJyB94qLi9PevXs1Y8aMJrkyadIkzZs3T5L00UcfyWaztTgu8xnv5aycgXebMmWKvvvd7zZ5VcJisWjVqlWSpNzcXJ06darFMT11nKEQgXapqKjQwYMHJdX9i/X/9e/fX2PHjpUk7d27t0Uxz549a5/4T506tcn5Bx98UP369ZPVatVnn33W1q7DTZyRM0BrkYcAGgoODr7rO9T17+OXlpa2eDLPfMa7OSNn0Lnde++99q+vX7/eoms8eZxhjQi0S1ZWlqxWq/z9/RUXF3fbNqNHj9aXX36pI0eOtChmZmampLp37aKiou4YMy8vT0eOHNHzzz/fpr7DPZyRMw2lpaUpJydHpaWlCg0N1bBhwzRx4kTFxMS0t+vwIs7OQ3i37OxsLVq0SMXFxQoKCpLFYtFTTz2lQYMGubtrcJIbN27Yv+7SpUuLrmE+07m1JWcaYj7T+WRkZEiSAgMDNWDAgBZd48njDIUItMuZM2ckSdHR0XesCvft27dR2+acPXu20XWOiImOwxk509DXX3/d6PO///1vvfXWW1q4cKFmz57d6njwTs7OQ3i3rKwsZWVl2T/v379fGzdu1IwZM7RkyRL5+vq6sXdwhj179kiSBg8erODg4BZdw3ymc2tLzjTEfKZzsNls9l2e1qxZI0lavHixgoKCWnS9J48zFCLQLlevXpV099V768/Vt3VkzGvXrrUoJjoOZ+SMJPXr109Lly7V2LFjFRMTI39/f508eVKJiYnau3ev1qxZo8DAQE2fPr19NwCv4Kw8hHeLjIzUggUL9P3vf1/33HOPgoODdebMGW3btk3bt29XcnKy/Pz89Mtf/tLdXYUDHT9+XNu3b5ckzZkzp8XXMZ/pvNqaMxLzmc4iKSmpyc5OcXFxWr16dau2ZvXkcYY1ItAuVVVVknTXd+Tqt4qpb+vImA0fe4NncEbOSNIrr7yin//85xoyZIhCQ0PVpUsXjRgxQn/961/1wgsvSJLWrl2rioqKdvQe3sJZeQjvNnXqVM2bN09xcXEKDw+Xv7+/LBaLfvvb32rx4sWSpOTkZOXn57u5p3CUS5cu6dVXX1VNTY0ee+wxPfXUUy2+lvlM59SenJGYz3QWUVFRGjVqlEaMGKGIiAiZTCZlZWVp165drSoYePI4QyEC7RIQECBJqq6uvmMbq9XaqK0jY7blnTu4lzNypjmvvfaazGazrl27pkOHDjkkJjybO/IQ3u3FF19UZGSkampqtH//fnd3Bw5QVlam2bNn6/z58xo2bJhWr17dquuZz3Q+7c2Z5jCf8R5PPvmk3n//fX3wwQf6z3/+o48//lgjRozQ7t27NWPGDNXW1rYojiePMxQi0C4teXS5JY8MNRQaGtrimPVt4TmckTPNCQkJsS8il5eX55CY8GzuyEN4N19fX40YMUIS44w3qKio0EsvvaRvvvlGgwYN0t///vdWv+fPfKZzcUTONIf5jPcaPHiwNm3apLCwMGVlZdnXGGmOJ48zFCLQLv3795cknT9//o6VuG+//bZR2+bUrxJ7twG2tTHRcTgjZ1qi/pG1mpoah8WE53JXHsK7Mc54h+vXr+vll19WZmam+vfvry1btigsLKzVcZjPdB6OypmWYJzxXsHBwXrggQckSSdOnGjRNZ48zlCIQLsMGTJEZrNZVqtVR48evW2b+q1o7rvvvhbFrP+NUn5+vi5evOiQmOg4nJEzzampqdHp06clSb169XJITHg2d+QhvF9OTo4kxhlPVlVVpVdeeUVff/21YmJilJSUpIiIiDbFYj7TOTgyZ5rDfMb71ReYWvpqhiePMxQi0C7BwcEaN26cJOmDDz5ocv7s2bP2d9ieeOKJFsUcMGCAYmNjJUk7duxocv6rr75SXl6ezGazHnnkkbZ2HW7ijJxpzo4dO1RWViY/Pz+NHTvWITHh2dyRh/BuBw4csBciHnroITf3Bm1RXV2tV199VV999ZWioqKUnJys3r17tzke8xnv5+icaQ7zGe9WWlqq9PR0SXW/MGkJTx5nKESg3ebOnSuTyaRdu3Zpx44dMgxDklRUVKTXXntNNptNjz76qAYPHtzouvj4eMXHx2vv3r1NYs6fP1+StHnz5kaLfp0+fVrLli2TJL3wwgsKDw931m3BiRydM6mpqfrTn/5k30u5ntVq1datW+3bI/3kJz9RZGSk824MHc60adMUHx+vpKSkJufamofwbnfKmZycHK1YsULZ2dmNjttsNu3evVuLFi2SJP3gBz9QXFycq7oLB6mtrdWiRYv0xRdfKCIiQsnJyerTp0+LrmU+0zk5I2eYz3i39PR0vf3227fdWenEiROaNWuWysrKFBUV1eSXIN44zpiM+pkX0A5JSUlavXq1DMNQ7969FRYWptzcXFmtVg0YMEDbtm1rkvwWi0WSlJCQoOeee65JzD/84Q9KTk6WJPXt21eBgYHKyclRbW2tRo8erS1btrCavQdzZM58+umnmjdvniSpZ8+eioqKkiSdOXNGlZWVkqQJEyZozZo19i2M4FkKCws1adIk+2er1arKykr5+fk1WgzspZde0uzZs+2f4+PjVVBQoPnz5+vVV19tErcteQjP4OicycrKssfr3r27oqOj5evrq2+//da+ENj999+vd955p8MtCIbmNSwmxcTE2P87cjvLly/X0KFD7Z+Zz3ROzsgZ5jPereHfb0REhCIjI+Xr66vCwkIVFxdLqtvWc9OmTU2eiPDGccbP3R2Ad5g5c6YsFosSExN19OhRXb58WdHR0XriiSc0Z84cBQUFtTrmr371K40cOVLbtm1TVlaWioqKNHDgQE2cOFEzZ86863656PgcmTPDhg3T3LlzlZmZqby8PJ05c0bV1dUKDw/XuHHj9Oyzzyo+Pt6JdwNnq62tVWlpaZPjNTU1jY63do9sZ4xd6BgcnTMxMTH6xS9+oczMTJ06dUp5eXmyWq3q1q2bHn74YT399NN6+umn5evr66A7gCvVb28nSQUFBSooKLhj27KyslbFZj7jnZyRM8xnvNvIkSP1+uuvKy0tTbm5uTp79qysVqtCQ0M1ZswYxcfHa/LkyW3abcUTxxmeiAAAAAAAAC7DGhEAAAAAAMBlKEQAAAAAAACXoRABAAAAAABchkIEAAAAAABwGQoRAAAAAADAZShEAAAAAAAAl6EQAQAAAAAAXIZCBAAAAAAAcBkKEQAAAAAAwGUoRAAAALiYxWKRxWJRWlqau7sCAIDL+bm7AwAAAOvXr9eGDRta3P7kyZNO7A0AAHAmChEAAKBD6dmzp7u7AAAAnIhCBAAA6FBSU1Pd3QUAAOBErBEBAAAAAABchiciAACAR4uPj1dBQYESEhL0+OOPa9OmTUpJSVFhYaG6du2q0aNH6+WXX9aIESPuGKO2tlY7d+7UP//5T508eVIVFRUKCwvTyJEjNX36dI0ZM+aufSgsLNTWrVuVmpqq/Px8VVdXKzIyUoMGDdKECRP05JNPKiAg4LbXlpeXa/Pmzdq3b5/Onz+vrl276r777tPcuXPv2mcAADwVhQgAAOAVrl27psmTJ+vMmTMym80KCAhQaWmpPvvsM33++edauXKlJk+e3OS6srIyzZ07V+np6ZIkX19fBQUFqbi4WPv27dO+ffv04osvasmSJbf9vh9//LFWrFihqqoqSZLZbFZQUJAKCwt17tw57d+/XxaLRUOGDGlybXFxsZ577jnl5eUpICBAPj4+Ki0t1YEDB5SamqqNGzdq3LhxDvwpAQDgfryaAQAAvMKGDRtUUlKitWvXKjMzUxkZGfrkk0/0wAMPyGaz6Te/+Y1OnDjR5Lpf//rXSk9Pl9ls1rJly5SRkaGvv/5aBw8e1I9//GNJUmJiot5///0m1x44cEBLly5VVVWVRo0apffee09Hjx5VWlqaDh8+rPfee09TpkyR2Wy+bZ9/97vfyWw2Kzk5WZmZmTp8+LA+/PBDDRgwQNXV1VqxYoVsNptjf1AAALiZyTAMw92dAAAAnVvD7Tub2zXjySef1LJly+yf61/NkKSkpCQ9+OCDjdrfuHFDzzzzjM6ePavx48frb3/7m/3ckSNHNGXKFEl1RYGpU6c2+X4LFizQvn37FBYWpi+++ML+ikVNTY0mTJig/Px8jR49WklJSfL392/R/VosFklSeHi4du/erR49ejQ6f/LkSU2cOFGStG3bNo0ePbpFcQEA8AQ8EQEAADqUS5cu3fVPeXn5ba8bNWpUkyKEJHXp0kWzZs2SJB08eFBlZWX2c5988okkqVevXnr++edvG3fhwoWSpCtXrjTa0SMtLU35+fmSpNdff73FRYiGpkyZ0qQIIdUVKu655x5JdUUJAAC8CWtEAACADqWt/+M9duzYZs/ZbDadOHHC/vn48eOSpDFjxsjH5/a/nxk4cKCioqJ08eJFHT9+XPHx8ZKkw4cPS5IiIiL0ne98p019vttilJGRkcrPz9fVq1fbFBsAgI6KJyIAAIBXiIqKatG5kpIS+9eXL19u9lqp7omJhu2luoUmJSk6Orr1nb0pKCjojuf8/Op+X1RTU9Pm+AAAdEQUIgAAANrAZDK5uwsAAHgkChEAAMArXLx4sUXnwsPD7V/Xr89w4cKFu8auP99wPYf6RTXPnz/f+s4CANCJUYgAAABeIS0trdlzPj4+Gjp0qP348OHD7efvtE3mqVOn7IWMhmtBjBo1SlLdKxrHjh1rX+cBAOhEKEQAAACvkJGRcdtiRFVVlRITEyVJ48aNU2hoqP3cU089JanuiYkPP/zwtnHXrVsnSQoLC9P3vvc9+/ExY8aoT58+kqSEhARZrVbH3AgAAF6OQgQAAPAKISEhWrBggfbu3Wtf4PHUqVOaM2eOTp8+LV9fXy1YsKDRNXFxcZowYYIkaeXKlXr33Xd1/fp1SXVPOixbtkx79+6VVLeNZ0BAgP1aX19fLV++XCaTSRkZGZo5c6b++9//2p+ssFqtSktL0+LFi5Wbm+v0+wcAwFOwfScAAOhQHnrooWbbrF+/3v5qRL358+dr+/btWrhwofz9/RUQEKCysjJJdQtLvvHGG7fdZnPVqlW6cuWK0tPTtXLlSiUkJCgoKEjXrl2TYRiSpBdffFHTpk1rcu348eO1evVqLV++XBkZGZo+fbr8/f0VGBio8vJye0Fk1qxZrf45AADgrShEAACADuXSpUvNtqmurm5yLDQ0VB999JE2bdqklJQUFRYWqnv37ho5cqRefvlljRw58raxQkJClJSUpJ07d2rXrl06efKkKisr1bNnT40aNUrTp0/XmDFj7tiXSZMm6f7779c//vEPpaam6vz586qqqlJ0dLRiY2P1+OOPa+DAgS3/AQAA4OVMRn2pHwAAwAPFx8eroKBACQkJeu6559zdHQAA0AzWiAAAAAAAAC5DIQIAAAAAALgMhQgAAAAAAOAyFCIAAAAAAIDLsFglAAAAAABwGZ6IAAAAAAAALkMhAgAAAAAAuAyFCAAAAAAA4DIUIgAAAAAAgMtQiAAAAAAAAC5DIQIAAAAAALgMhQgAAAAAAOAyFCIAAAAAAIDLUIgAAAAAAAAu838V19hj7Hl+sAAAAABJRU5ErkJggg==\n" }, "metadata": {} } ] } ] }