DiceyTech_CaptionGenerator.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1hqpnCEp4pac0DrQr5Qq_t3GeqXeCZ5Iw",
      "authorship_tag": "ABX9TyMoiZkuEcQI9xh2IuCyP+Vk",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "96059783bc904a70b9e6f70423135913": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1bca2b3a44014e90bc7b349e8058044f",
              "IPY_MODEL_3df51fb18d024ca8b44a5b446950f791",
              "IPY_MODEL_4cbaafd2e01746fb809f565ca6fdf4aa"
            ],
            "layout": "IPY_MODEL_f168d7431e3f45718722e1573d14e5ea"
          }
        },
        "1bca2b3a44014e90bc7b349e8058044f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e88ffa2827134011a642e4df936a8293",
            "placeholder": "​",
            "style": "IPY_MODEL_f26bd12cb10e4d03a74c77365aa1e90c",
            "value": "100%"
          }
        },
        "3df51fb18d024ca8b44a5b446950f791": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31eddee11e7047c5b9c390373355283e",
            "max": 3973,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_91a62bf41f88492a9c2e3fc01311e02a",
            "value": 3973
          }
        },
        "4cbaafd2e01746fb809f565ca6fdf4aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_568a9c97a11e46ec95e5452f022a915b",
            "placeholder": "​",
            "style": "IPY_MODEL_408c830e34d644bf99050e6da98d86cb",
            "value": " 3973/3973 [1:55:00&lt;00:00,  1.58it/s, lr=0.0001, train_loss=1.49]"
          }
        },
        "f168d7431e3f45718722e1573d14e5ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e88ffa2827134011a642e4df936a8293": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f26bd12cb10e4d03a74c77365aa1e90c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "31eddee11e7047c5b9c390373355283e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91a62bf41f88492a9c2e3fc01311e02a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "568a9c97a11e46ec95e5452f022a915b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "408c830e34d644bf99050e6da98d86cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c56b109744a447cd80d9c6e3af01853b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5ccae47fe6d343aca3cecdb974eb89f5",
              "IPY_MODEL_83b8a697173d4a91975725e41841c098",
              "IPY_MODEL_4e6eb0111e5e467fa88a75cf12d34b7b"
            ],
            "layout": "IPY_MODEL_4957d9f7ed334cc6a794c7474494cf1d"
          }
        },
        "5ccae47fe6d343aca3cecdb974eb89f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_08c52cae855b422e87fe3964261ca522",
            "placeholder": "​",
            "style": "IPY_MODEL_0b47c63183e54c0f9fc57682ae85c9d1",
            "value": "100%"
          }
        },
        "83b8a697173d4a91975725e41841c098": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ee13477022fb46ba8f97991a057b361e",
            "max": 994,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_73329889cbe74faf879453db36dff3b4",
            "value": 994
          }
        },
        "4e6eb0111e5e467fa88a75cf12d34b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a39898f884144c2faa163790e19e97d2",
            "placeholder": "​",
            "style": "IPY_MODEL_3e2adec89f314bcf94e31db8486d802e",
            "value": " 994/994 [04:47&lt;00:00,  4.38it/s, valid_loss=0.626]"
          }
        },
        "4957d9f7ed334cc6a794c7474494cf1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08c52cae855b422e87fe3964261ca522": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b47c63183e54c0f9fc57682ae85c9d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ee13477022fb46ba8f97991a057b361e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73329889cbe74faf879453db36dff3b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a39898f884144c2faa163790e19e97d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e2adec89f314bcf94e31db8486d802e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd7d9fdbe0a44c5a8190057671b8ec65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e4f264ff882a48cf825a00d5711f2b4c",
              "IPY_MODEL_b7c43069f35145e4aca71d6b82acae83",
              "IPY_MODEL_8b5fe3fbf9af499ab52868dd0c75e4be"
            ],
            "layout": "IPY_MODEL_ef1e978cefca45079bc60118fcfb6a7d"
          }
        },
        "e4f264ff882a48cf825a00d5711f2b4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8c40849b0da46b7871996127a3d2648",
            "placeholder": "​",
            "style": "IPY_MODEL_a1f095a881254b8b80ef3bebaffcf236",
            "value": "100%"
          }
        },
        "b7c43069f35145e4aca71d6b82acae83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce5f2007ac854b40a1c72e400e5b0c3a",
            "max": 3973,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_77ee135711f544d0b5868b5a3e03c63a",
            "value": 3973
          }
        },
        "8b5fe3fbf9af499ab52868dd0c75e4be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d32395e5f3f04e7a8b6aa874fcac1d0c",
            "placeholder": "​",
            "style": "IPY_MODEL_bb46e39553c44cdc95b7715f5480f27d",
            "value": " 3973/3973 [39:32&lt;00:00,  1.74it/s, lr=0.0001, train_loss=0.518]"
          }
        },
        "ef1e978cefca45079bc60118fcfb6a7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8c40849b0da46b7871996127a3d2648": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1f095a881254b8b80ef3bebaffcf236": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ce5f2007ac854b40a1c72e400e5b0c3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77ee135711f544d0b5868b5a3e03c63a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d32395e5f3f04e7a8b6aa874fcac1d0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb46e39553c44cdc95b7715f5480f27d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20116dad63204bb6a472b5831ef3a73f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b6ca96b1548346c3bd1d1f017454dd7e",
              "IPY_MODEL_c8d188df0e6a40b095fd8d0b95a29dff",
              "IPY_MODEL_a17560933a66428e97be1ddae3bd5698"
            ],
            "layout": "IPY_MODEL_fe3823794a384a08be0288bc95a6f76e"
          }
        },
        "b6ca96b1548346c3bd1d1f017454dd7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a1b2af4ca14d4d3db73c075f9c0f7f96",
            "placeholder": "​",
            "style": "IPY_MODEL_8c44a09199e343db8892bf03b0bdb5f6",
            "value": "100%"
          }
        },
        "c8d188df0e6a40b095fd8d0b95a29dff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_caf8373ea7594cecacde14ab217cc9f2",
            "max": 994,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9371a42ee10e434bb5127fe1edbaed62",
            "value": 994
          }
        },
        "a17560933a66428e97be1ddae3bd5698": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9a9ef0b750234633ba27652efa59057b",
            "placeholder": "​",
            "style": "IPY_MODEL_34a74fb0b32e47a0be6d0d040dd68d71",
            "value": " 994/994 [04:47&lt;00:00,  4.02it/s, valid_loss=0.43]"
          }
        },
        "fe3823794a384a08be0288bc95a6f76e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1b2af4ca14d4d3db73c075f9c0f7f96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c44a09199e343db8892bf03b0bdb5f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "caf8373ea7594cecacde14ab217cc9f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9371a42ee10e434bb5127fe1edbaed62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9a9ef0b750234633ba27652efa59057b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "34a74fb0b32e47a0be6d0d040dd68d71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/emms204/DiceyTech-CaptionGenerator/blob/main/DiceyTech_CaptionGenerator.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ASzZqDaVqcDj",
        "outputId": "842a93b4-5351-486c-eb27-59c3cd82ebb2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 8359M  100 8359M    0     0   139M      0  0:01:00  0:01:00 --:--:--  165M\n"
          ]
        }
      ],
      "source": [
        "#!curl \"https://storage.googleapis.com/kaggle-data-sets/31296/39911/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20221230%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20221230T105358Z&X-Goog-Expires=259200&X-Goog-SignedHeaders=host&X-Goog-Signature=9eb6c3635794174f7d3e3f1efd445b5e96193fb2c20524b32ac01da7de01f78597f9907d9304c3f02a7dc4d7718835a0e9f0d75d164cbea477f2ca6e9a9d5bc442b6368d48f1a39ce6a0baf31f10cfe2ae87e087dd959df9e2b4590b8b171546b119cd495b264607afad722300b009daa112437a6fbc6298eacfdc0761423e58253877c5cedfb02f19ca14efff45bef51d47a45eebc269a64b44c3c35801b492b200ac27afc93f59af8a6116846165a5f1a2aeb42871441209593f16653fff4a74bd868c88ba9a6292c0db2318190cefc3e5e2e602d3e8c849f3700f3e4e9c132b5afae09f56121cedb0897fe5f1880b08cb813bc48dc22cc7d44d5e896d1fc8\" >Flickr.zip"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir('/content/drive/MyDrive/Flickr Image Dataset')"
      ],
      "metadata": {
        "id": "JssfLOQ8q21r"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# !unzip -q /content/Flickr.zip"
      ],
      "metadata": {
        "id": "89C70x-Wr1xT"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install timm -q\n",
        "!pip install transformers -q"
      ],
      "metadata": {
        "id": "Z--faSZkr_g2"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import cv2\n",
        "import gc\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import itertools\n",
        "from tqdm.autonotebook import tqdm\n",
        "import albumentations as A\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import timm\n",
        "from transformers import DistilBertModel, DistilBertConfig, DistilBertTokenizer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-bKsufgE2SC0",
        "outputId": "7216ff8d-abbf-477c-c80a-043cdf99b11c"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-4-6272a29e912d>:7: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"flickr30k_images/results.csv\", delimiter=\"|\")\n",
        "df.columns = ['image', 'caption_number', 'caption']\n",
        "df['caption'] = df['caption'].str.lstrip()\n",
        "df['caption_number'] = df['caption_number'].str.lstrip()\n",
        "df.loc[19999, 'caption_number'] = \"4\"\n",
        "df.loc[19999, 'caption'] = \"A dog runs across the grass .\"\n",
        "ids = [id_ for id_ in range(len(df) // 5) for i in range(5)]\n",
        "df['id'] = ids\n",
        "df.to_csv(\"captions.csv\", index=False)\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "4Q9yDZiW2sfg",
        "outputId": "437153d1-c345-426c-f540-8e15c39dcf6f"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            image caption_number  \\\n",
              "0  1000092795.jpg              0   \n",
              "1  1000092795.jpg              1   \n",
              "2  1000092795.jpg              2   \n",
              "3  1000092795.jpg              3   \n",
              "4  1000092795.jpg              4   \n",
              "\n",
              "                                             caption  id  \n",
              "0  Two young guys with shaggy hair look at their ...   0  \n",
              "1  Two young , White males are outside near many ...   0  \n",
              "2   Two men in green shirts are standing in a yard .   0  \n",
              "3       A man in a blue shirt standing in a garden .   0  \n",
              "4            Two friends enjoy time spent together .   0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-64e3778a-6a44-4a86-bf1a-5d3a7761d0c9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>caption_number</th>\n",
              "      <th>caption</th>\n",
              "      <th>id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1000092795.jpg</td>\n",
              "      <td>0</td>\n",
              "      <td>Two young guys with shaggy hair look at their ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1000092795.jpg</td>\n",
              "      <td>1</td>\n",
              "      <td>Two young , White males are outside near many ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1000092795.jpg</td>\n",
              "      <td>2</td>\n",
              "      <td>Two men in green shirts are standing in a yard .</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1000092795.jpg</td>\n",
              "      <td>3</td>\n",
              "      <td>A man in a blue shirt standing in a garden .</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1000092795.jpg</td>\n",
              "      <td>4</td>\n",
              "      <td>Two friends enjoy time spent together .</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-64e3778a-6a44-4a86-bf1a-5d3a7761d0c9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-64e3778a-6a44-4a86-bf1a-5d3a7761d0c9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-64e3778a-6a44-4a86-bf1a-5d3a7761d0c9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Config"
      ],
      "metadata": {
        "id": "saIAig1WDMaR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CFG:\n",
        "  debug = False\n",
        "  image_path = \"flickr30k_images/flickr30k_images\"\n",
        "  captions_path = \".\"\n",
        "  batch_size = 32\n",
        "  num_workers = 2\n",
        "  head_lr = 1e-3\n",
        "  image_encoder_lr = 1e-4\n",
        "  text_encoder_lr = 1e-5\n",
        "  weight_decay = 1e-3\n",
        "  patience = 1\n",
        "  factor = 0.8\n",
        "  epochs = 2\n",
        "  device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "  model_name = 'resnet50'\n",
        "  image_embedding = 2048\n",
        "  text_encoder_model = \"distilbert-base-uncased\"\n",
        "  text_embedding = 768\n",
        "  text_tokenizer = \"distilbert-base-uncased\"\n",
        "  max_length = 200\n",
        "\n",
        "  pretrained = True # for both image encoder and text encoder\n",
        "  trainable = True # for both image encoder and text encoder\n",
        "  temperature = 1.0\n",
        "\n",
        "  # image size\n",
        "  size = 224\n",
        "\n",
        "  # for projection head; used for both image and text encoders\n",
        "  num_projection_layers = 1\n",
        "  projection_dim = 256 \n",
        "  dropout = 0.1\n"
      ],
      "metadata": {
        "id": "mZJleeYI3iEx"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#UTILS"
      ],
      "metadata": {
        "id": "tnvlfGeiDHF6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class AvgMeter:\n",
        "  def __init__(self, name=\"Metric\"):\n",
        "    self.name = name\n",
        "    self.reset()\n",
        "\n",
        "  def reset(self):\n",
        "    self.avg, self.sum, self.count = [0] * 3\n",
        "\n",
        "  def update(self, val, count=1):\n",
        "    self.count += count\n",
        "    self.sum += val * count\n",
        "    self.avg = self.sum / self.count\n",
        "\n",
        "  def __repr__(self):\n",
        "    text = f\"{self.name}: {self.avg:.4f}\"\n",
        "    return text\n",
        "\n",
        "def get_lr(optimizer):\n",
        "  for param_group in optimizer.param_groups:\n",
        "    return param_group[\"lr\"]\n",
        "\n"
      ],
      "metadata": {
        "id": "orYOJVhB3Bne"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Dataset"
      ],
      "metadata": {
        "id": "XetAshajDA5k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class CLIPDataset(torch.utils.data.Dataset):\n",
        "  def __init__(self, image_filenames,captions,tokenizer,transforms):\n",
        "    \"\"\"image_filenames and captions must have the same length; so, if there are \n",
        "    multiple captions for each image, the image_filenames must have repetitive file names\"\"\"\n",
        "    self.image_filenames = image_filenames\n",
        "    self.captions = list(captions)\n",
        "    self.encoded_captions = tokenizer(list(captions),padding=True,truncation=True,max_length=CFG.max_length)\n",
        "    self.transforms = transforms\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    item = {key:torch.tensor(values[idx]) for key,values in self.encoded_captions.items()}\n",
        "    image = cv2.imread(f\"{CFG.image_path}/{self.image_filenames[idx]}\")\n",
        "    image = self.transforms(image=image)['image']\n",
        "    item['image'] = torch.tensor(image).permute(2,0,1).float()\n",
        "    item['caption'] = self.captions[idx]\n",
        "    return item\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.captions)"
      ],
      "metadata": {
        "id": "Xu9c6JC15-nj"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_transforms(mode=\"train\"):\n",
        "  if mode == \"train\":\n",
        "    return A.Compose([\n",
        "        A.Resize(CFG.size, CFG.size,always_apply=True),\n",
        "        A.Normalize(max_pixel_value=255.0,always_apply=True),])\n",
        "  else:\n",
        "    return A.Compose([\n",
        "        A.Resize(CFG.size, CFG.size,always_apply=True),\n",
        "        A.Normalize(max_pixel_value=255.0,always_apply=True),])"
      ],
      "metadata": {
        "id": "ujQsjWPiALyg"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Image Encoder"
      ],
      "metadata": {
        "id": "UK4vvSnPDeFD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ImageEncoder(nn.Module):\n",
        "  \"\"\"Encode Images to a fixed size vector\"\"\"\n",
        "  def __init__(self,model_name=CFG.model_name,pretrained=CFG.pretrained,trainable=CFG.trainable):\n",
        "    super().__init__()\n",
        "    self.model = timm.create_model(model_name,pretrained,num_classes=0,global_pool=\"avg\")\n",
        "    for p in self.model.parameters():\n",
        "      p.requires_grad = trainable\n",
        "\n",
        "  def forward(self, x):\n",
        "    return self.model(x)"
      ],
      "metadata": {
        "id": "4w_SOjAVDdOS"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Text Encoder"
      ],
      "metadata": {
        "id": "m6SzedluEIma"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TextEncoder(nn.Module):\n",
        "  def __init__(self, model_name=CFG.text_encoder_model, pretrained=CFG.pretrained, trainable=CFG.trainable):\n",
        "    super().__init__()\n",
        "    if pretrained:\n",
        "      self.model = DistilBertModel.from_pretrained(model_name)\n",
        "    else:\n",
        "      self.model = DistilBertModel(config=DistilBertConfig())\n",
        "    # for p in self.model.parameters():\n",
        "    #   p.requires_grad = trainable\n",
        "\n",
        "    #we are using the CLS token hidden representation as the sentence's embedding\n",
        "    self.target_token_idx = 0\n",
        "\n",
        "  def forward(self, input_ids, attention_mask):\n",
        "    output = self.model(input_ids=input_ids, attention_mask=attention_mask)\n",
        "    last_hidden_state = output.last_hidden_state\n",
        "    return last_hidden_state[:,self.target_token_idx,:]"
      ],
      "metadata": {
        "id": "JQlFRK-dEHdL"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Projection Head"
      ],
      "metadata": {
        "id": "XIjkJmMmFQxR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ProjectionHead(nn.Module):\n",
        "  def __init__(self,embedding_dim,projection_dim=CFG.projection_dim,dropout=CFG.dropout):\n",
        "    super().__init__()\n",
        "    self.projection = nn.Linear(embedding_dim,projection_dim)\n",
        "    self.gelu = nn.GELU()\n",
        "    self.fc = nn.Linear(projection_dim,projection_dim)\n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    self.layer_norm = nn.LayerNorm(projection_dim)\n",
        "\n",
        "  def forward(self, x):\n",
        "    projected = self.projection(x)\n",
        "    x = self.gelu(projected)\n",
        "    x = self.fc(x)\n",
        "    x = self.dropout(x)\n",
        "    x = x+projected\n",
        "    x = self.layer_norm(x)\n",
        "    return x"
      ],
      "metadata": {
        "id": "xNVf7_PHFP9L"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#CLIP"
      ],
      "metadata": {
        "id": "DV0N09l41u-5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def cross_entropy(preds, targets, reduction='none'):\n",
        "  log_softmax = nn.LogSoftmax(dim=-1)\n",
        "  loss = (-targets * log_softmax(preds)).sum(1)\n",
        "  if reduction =='none':return loss\n",
        "  elif reduction == \"mean\": return loss.mean()"
      ],
      "metadata": {
        "id": "KwvaWh8K4Z4N"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class CLIPModel(nn.Module):\n",
        "  def __init__(self,temperature=CFG.temperature,image_embedding=CFG.image_embedding,text_embedding=CFG.text_embedding):\n",
        "    super().__init__()\n",
        "    self.image_encoder = ImageEncoder()\n",
        "    self.text_encoder = TextEncoder()\n",
        "    self.image_projection = ProjectionHead(embedding_dim=image_embedding)\n",
        "    self.text_projection = ProjectionHead(embedding_dim=text_embedding)\n",
        "    self.temperature = temperature\n",
        "\n",
        "  def forward(self,batch):\n",
        "    #Getting Image and and Text Features\n",
        "    image_features = self.image_encoder(batch[\"image\"])\n",
        "    text_features = self.text_encoder(input_ids=batch[\"input_ids\"],attention_mask=batch[\"attention_mask\"])\n",
        "    #Getting Image and Text Embeddings (with same dimension)\n",
        "    image_embeddings = self.image_projection(image_features)\n",
        "    text_embeddings = self.text_projection(text_features)\n",
        "    #Calculating the Loss\n",
        "    logits = (text_embeddings @ image_embeddings.T)/self.temperature\n",
        "    images_similarity = image_embeddings @ image_embeddings.T\n",
        "    texts_similarity = text_embeddings @ text_embeddings.T\n",
        "    targets = torch.softmax((images_similarity+texts_similarity)/2*self.temperature,dim=-1)\n",
        "    texts_loss = cross_entropy(logits, targets, reduction='none')\n",
        "    images_loss = cross_entropy(logits.T,targets.T,reduction='none')\n",
        "    loss = (images_loss + texts_loss) / 2.0\n",
        "    return loss.mean()\n",
        "\n"
      ],
      "metadata": {
        "id": "7P36W_FF1tvf"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def make_train_valid_dfs():\n",
        "  dataframe = pd.read_csv(f\"{CFG.captions_path}/captions.csv\")\n",
        "  train_df, valid_df =train_test_split(dataframe, test_size=0.2,shuffle=True)\n",
        "  return train_df.reset_index(drop=True),valid_df.reset_index(drop=True)\n",
        "\n",
        "def build_loaders(dataframe,tokenizer,mode):\n",
        "  transforms = get_transforms(mode=mode)\n",
        "  dataset = CLIPDataset(dataframe[\"image\"].values,dataframe[\"caption\"].values,tokenizer=tokenizer,transforms=transforms)\n",
        "  dataloader = torch.utils.data.DataLoader(dataset,batch_size=CFG.batch_size,num_workers=CFG.num_workers,\n",
        "                                           shuffle=True if mode==\"train\" else False)\n",
        "  return dataloader"
      ],
      "metadata": {
        "id": "HGYo7Ure5TUY"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_epoch(model,train_loader,optimizer,lr_scheduler,step):\n",
        "  loss_meter = AvgMeter()\n",
        "  tqdm_object = tqdm(train_loader,total=len(train_loader))\n",
        "  for batch in tqdm_object:\n",
        "    batch = {k: v.to(CFG.device) for k,v in batch.items() if k != \"caption\"}\n",
        "    loss = model(batch)\n",
        "    optimizer.zero_grad()\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    if step == 'batch':\n",
        "      lr_scheduler.step()\n",
        "\n",
        "    count = batch[\"image\"].size(0)\n",
        "    loss_meter.update(loss.item(),count)\n",
        "    tqdm_object.set_postfix(train_loss=loss_meter.avg,lr=get_lr(optimizer))\n",
        "  return loss_meter\n",
        "\n",
        "def valid_epoch(model,valid_loader):\n",
        "  loss_meter = AvgMeter()\n",
        "  tqdm_object = tqdm(valid_loader,total=len(valid_loader))\n",
        "  for batch in tqdm_object:\n",
        "    batch = {k: v.to(CFG.device)for k,v in batch.items() if k != \"caption\"}\n",
        "    loss = model(batch)\n",
        "    count = batch[\"image\"].size(0)\n",
        "    loss_meter.update(loss.item(),count)\n",
        "    tqdm_object.set_postfix(valid_loss=loss_meter.avg)\n",
        "  return loss_meter\n",
        "\n"
      ],
      "metadata": {
        "id": "m0n4Rvgf6ej9"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "  train_df,valid_df = make_train_valid_dfs()\n",
        "  tokenizer = DistilBertTokenizer.from_pretrained(CFG.text_tokenizer)\n",
        "  train_loader = build_loaders(train_df,tokenizer,mode=\"train\")\n",
        "  valid_loader = build_loaders(valid_df,tokenizer,mode=\"valid\")\n",
        "\n",
        "  model = CLIPModel().to(CFG.device)\n",
        "  params = [\n",
        "      {\"params\":model.image_encoder.parameters(),\"lr\":CFG.image_encoder_lr},\n",
        "      {\"params\":model.text_encoder.parameters(),\"lr\":CFG.text_encoder_lr},\n",
        "      {\"params\":itertools.chain(model.image_projection.parameters(),model.text_projection.parameters()),\n",
        "       \"lr\":CFG.head_lr,\"weight_decay\":CFG.weight_decay}\n",
        "  ]\n",
        "  optimizer = torch.optim.AdamW(params,weight_decay=0.)\n",
        "  lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n",
        "      optimizer, mode='min',patience=CFG.patience,factor=CFG.factor\n",
        "  )\n",
        "  step = \"\"\n",
        "  best_loss = float('inf')\n",
        "  for epoch in range(CFG.epochs):\n",
        "    print(f\"Epoch: {epoch + 1}\")\n",
        "    model.train()\n",
        "    train_loss = train_epoch(model,train_loader,optimizer,lr_scheduler,step)\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "      valid_loss = valid_epoch(model,valid_loader)\n",
        "    if valid_loss.avg < best_loss:\n",
        "      best_loss = valid_loss.avg\n",
        "      torch.save(model.state_dict(),\"best.pth\")\n",
        "      print(\"Saved Best Model!\")\n",
        "\n",
        "    lr_scheduler.step(valid_loss.avg)\n",
        "\n",
        "      \n"
      ],
      "metadata": {
        "id": "nD-ao74mGyK6"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "main()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287,
          "referenced_widgets": [
            "96059783bc904a70b9e6f70423135913",
            "1bca2b3a44014e90bc7b349e8058044f",
            "3df51fb18d024ca8b44a5b446950f791",
            "4cbaafd2e01746fb809f565ca6fdf4aa",
            "f168d7431e3f45718722e1573d14e5ea",
            "e88ffa2827134011a642e4df936a8293",
            "f26bd12cb10e4d03a74c77365aa1e90c",
            "31eddee11e7047c5b9c390373355283e",
            "91a62bf41f88492a9c2e3fc01311e02a",
            "568a9c97a11e46ec95e5452f022a915b",
            "408c830e34d644bf99050e6da98d86cb",
            "c56b109744a447cd80d9c6e3af01853b",
            "5ccae47fe6d343aca3cecdb974eb89f5",
            "83b8a697173d4a91975725e41841c098",
            "4e6eb0111e5e467fa88a75cf12d34b7b",
            "4957d9f7ed334cc6a794c7474494cf1d",
            "08c52cae855b422e87fe3964261ca522",
            "0b47c63183e54c0f9fc57682ae85c9d1",
            "ee13477022fb46ba8f97991a057b361e",
            "73329889cbe74faf879453db36dff3b4",
            "a39898f884144c2faa163790e19e97d2",
            "3e2adec89f314bcf94e31db8486d802e",
            "bd7d9fdbe0a44c5a8190057671b8ec65",
            "e4f264ff882a48cf825a00d5711f2b4c",
            "b7c43069f35145e4aca71d6b82acae83",
            "8b5fe3fbf9af499ab52868dd0c75e4be",
            "ef1e978cefca45079bc60118fcfb6a7d",
            "c8c40849b0da46b7871996127a3d2648",
            "a1f095a881254b8b80ef3bebaffcf236",
            "ce5f2007ac854b40a1c72e400e5b0c3a",
            "77ee135711f544d0b5868b5a3e03c63a",
            "d32395e5f3f04e7a8b6aa874fcac1d0c",
            "bb46e39553c44cdc95b7715f5480f27d",
            "20116dad63204bb6a472b5831ef3a73f",
            "b6ca96b1548346c3bd1d1f017454dd7e",
            "c8d188df0e6a40b095fd8d0b95a29dff",
            "a17560933a66428e97be1ddae3bd5698",
            "fe3823794a384a08be0288bc95a6f76e",
            "a1b2af4ca14d4d3db73c075f9c0f7f96",
            "8c44a09199e343db8892bf03b0bdb5f6",
            "caf8373ea7594cecacde14ab217cc9f2",
            "9371a42ee10e434bb5127fe1edbaed62",
            "9a9ef0b750234633ba27652efa59057b",
            "34a74fb0b32e47a0be6d0d040dd68d71"
          ]
        },
        "id": "l06wWRA7kHqp",
        "outputId": "ba2375db-bc26-45c8-d70a-e49c7c65fd6b"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_projector.weight', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3973 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "96059783bc904a70b9e6f70423135913"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/994 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c56b109744a447cd80d9c6e3af01853b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saved Best Model!\n",
            "Epoch: 2\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3973 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd7d9fdbe0a44c5a8190057671b8ec65"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/994 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "20116dad63204bb6a472b5831ef3a73f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saved Best Model!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Inference"
      ],
      "metadata": {
        "id": "7erO5tqpAb8b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_image_embeddings(valid_df, model_path):\n",
        "  tokenizer = DistilBertTokenizer.from_pretrained(CFG.text_tokenizer)\n",
        "  valid_loader = build_loaders(valid_df, tokenizer, mode=\"valid\")\n",
        "    \n",
        "  model = CLIPModel().to(CFG.device)\n",
        "  model.load_state_dict(torch.load(model_path, map_location=CFG.device))\n",
        "  model.eval()\n",
        "  \n",
        "  valid_image_embeddings = []\n",
        "  with torch.no_grad():\n",
        "      for batch in tqdm(valid_loader):\n",
        "          image_features = model.image_encoder(batch[\"image\"].to(CFG.device))\n",
        "          image_embeddings = model.image_projection(image_features)\n",
        "          valid_image_embeddings.append(image_embeddings)\n",
        "  return model, torch.cat(valid_image_embeddings)"
      ],
      "metadata": {
        "id": "MQ6kLuKWAX5D"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "_, valid_df = make_train_valid_dfs()\n",
        "model, image_embeddings = get_image_embeddings(valid_df, \"best.pth\")"
      ],
      "metadata": {
        "id": "JZoK32NCBFtK"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "3VAK7ol9BK7Y"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}