ABSA_Normal_ML_final.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G1x4BGuxIspC",
        "outputId": "1112197c-6395-4df9-a76a-9c7c1dbc4b05"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'ABSA-VLSP2018'...\n",
            "remote: Enumerating objects: 76, done.\u001b[K\n",
            "remote: Counting objects: 100% (76/76), done.\u001b[K\n",
            "remote: Compressing objects: 100% (55/55), done.\u001b[K\n",
            "remote: Total 76 (delta 40), reused 43 (delta 17), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (76/76), 712.48 KiB | 6.04 MiB/s, done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://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@github.com/htuannn/ABSA-VLSP2018"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MuO55XVb6AW0",
        "outputId": "a0abe0a8-eb10-41f8-8263-bd2201769d57"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/ABSA-VLSP2018\n"
          ]
        }
      ],
      "source": [
        "%cd /content/ABSA-VLSP2018"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "cell_id": "00001-73dc905a-f8e8-4f5a-b542-50a626b67551",
        "deepnote_cell_type": "markdown",
        "id": "NZ2MidPrPrae",
        "tags": []
      },
      "source": [
        "# Read data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ERItqiDmmm5m",
        "outputId": "67329cf7-671e-45e9-b335-502704b4eac3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m356.6/356.6 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m390.6/390.6 kB\u001b[0m \u001b[31m34.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.2/19.2 MB\u001b[0m \u001b[31m84.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m27.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m993.5/993.5 kB\u001b[0m \u001b[31m75.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m657.8/657.8 kB\u001b[0m \u001b[31m54.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.7/78.7 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for flashtext (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install -Uq emoji \\\n",
        "                 optuna \\\n",
        "                 flashtext \\\n",
        "                 underthesea \\\n",
        "                 scikit-learn"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "KM0ohCtvRaHZ"
      },
      "source": [
        "## mo2ml - Multioutput to multilabel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "iZif1GwORnKq"
      },
      "outputs": [],
      "source": [
        "aspects = ['HOTEL_ROOM#QUALITY',\n",
        "           'HOTEL_ROOM#PRICES',\n",
        "           'HOTEL_ROOM#CLEANLINESS',\n",
        "           'HOTEL_ROOM#GENERAL',\n",
        "           'FACILITY#GENERAL',\n",
        "           'FOOD#QUALITY',\n",
        "           'FOOD#OPTION',\n",
        "           'SERVICE#GENERAL',\n",
        "           'LOCATION#GENERAL']\n",
        "\n",
        "sentiments = ['-', 'o', '+']\n",
        "\n",
        "def mo2ml(y):\n",
        "    \"\"\"Convert multi-output to multi-label data\n",
        "    \"\"\"\n",
        "    newcols = [f'{a} {s}' for a in aspects for s in sentiments]\n",
        "\n",
        "    nrows, ncols = len(y), len(newcols)\n",
        "    ml = pd.DataFrame(np.zeros((nrows, ncols), dtype='bool'),\n",
        "                      columns=newcols)\n",
        "\n",
        "    for i, a in enumerate(aspects):\n",
        "        for j in range(1, 4):\n",
        "            indices = y[a] == j\n",
        "            ml.iloc[indices, i * 3 + j - 1] = True\n",
        "\n",
        "    return ml"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "jhbl0ne_6uET"
      },
      "source": [
        "## mo2df - Multioutput to DataFrame"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "1zKR4D6_7cOd"
      },
      "outputs": [],
      "source": [
        "from utils import *\n",
        "\n",
        "def mo2df(y):\n",
        "    if isinstance(y, pd.DataFrame):\n",
        "        return y\n",
        "    return pd.DataFrame(y, columns=aspects)\n",
        "\n",
        "def init_data(df):\n",
        "    X = df.pop('review')\n",
        "    y = df.replace({np.nan: 0,\n",
        "                    'Negative': 1,\n",
        "                    'Neutral': 2,\n",
        "                    'Positive': 3}).astype(np.uint8)\n",
        "\n",
        "    print('X.shape:', X.shape, 'y.shape:', y.shape)\n",
        "    return X, y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LQy6J2nk7lqo",
        "outputId": "d9c70d7a-b269-4924-8ab1-268086e88376"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "### Loading config {'aspect': ['HOTEL_ROOM#QUALITY', 'HOTEL_ROOM#PRICES', 'HOTEL_ROOM#CLEANLINESS', 'HOTEL_ROOM#GENERAL', 'FACILITY#GENERAL', 'FOOD#QUALITY', 'FOOD#OPTION', 'SERVICE#GENERAL', 'LOCATION#GENERAL'], 'labels': ['negative', 'neutral', 'positive'], 'pretrained': 'vinai/phobert-base-v2', 'use_lstm': True, 'num_layers_lstm': 2, 'word_embedding_dim': 512, 'num_embeddings': 256, 'batch_size': 256, 'num_workers': 2, 'num_epochs': 30, 'train_file': 'data/hotel_absa_train.txt', 'val_file': 'data/hotel_absa_val.txt', 'test_file': 'data/hotel_absa_test.txt', 'freeze_embedder': True, 'acd_warmup': 30, 'acd_only': False, 'acsc_only': False, 'lr': 5e-05, 'weight_decay': 1e-05, 'acd_loss_weight': 1.0, 'acsc_loss_weight': 1.0, 'saved_model': 'log/model_absa.bin', 'model_save_path': 'log/model_absa.bin', 'device': device(type='cuda', index=0)}\n"
          ]
        }
      ],
      "source": [
        "with open('config/absa_model.yaml') as f:\n",
        "  cfg = yaml.load(f, Loader=yaml.SafeLoader)\n",
        "  cfg['device']= torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "  print(f\"### Loading config {cfg}\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LgPJvtmZ7gbd",
        "outputId": "e921fd7e-27f5-4130-9f94-ab60ad1b236c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "X.shape: (1688,) y.shape: (1688, 9)\n",
            "X.shape: (252,) y.shape: (252, 9)\n",
            "X.shape: (507,) y.shape: (507, 9)\n"
          ]
        }
      ],
      "source": [
        "train = txt2df(cfg['train_file'], cfg['aspect'])\n",
        "dev = txt2df(cfg['val_file'], cfg['aspect'])\n",
        "test = txt2df(cfg['test_file'], cfg['aspect'])\n",
        "\n",
        "Xtrain, ytrain = init_data(train)\n",
        "Xdev,   ydev   = init_data(dev)\n",
        "Xtest,  ytest  = init_data(test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "dJryQqeGleTl"
      },
      "outputs": [],
      "source": [
        "# y target for evaluation\n",
        "ytrain_ml = mo2ml(ytrain)\n",
        "ydev_ml   = mo2ml(ydev)\n",
        "ytest_ml  = mo2ml(ytest)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "-Xt3zWvUwnFQ"
      },
      "source": [
        "#Preprocess data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "5xa5t4csleTm"
      },
      "outputs": [],
      "source": [
        "from preprocess import preprocess_fn\n",
        "\n",
        "xtrain = Xtrain.apply(preprocess_fn)\n",
        "xdev   = Xdev.apply(preprocess_fn)\n",
        "xtest  = Xtest.apply(preprocess_fn)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "cell_id": "00003-b0c4c0b2-689b-4f0b-9bb7-f78acf400fba",
        "deepnote_cell_type": "markdown",
        "id": "gtxqTCE_Prat",
        "tags": []
      },
      "source": [
        "# EDA"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "8y6zihw1FZHx"
      },
      "source": [
        "## References\n",
        "- https://developers.google.com/machine-learning/guides/text-classification/step-2\n",
        "- https://github.com/google/eng-edu/blob/main/ml/guides/text_classification/explore_data.py"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "0AJu2t0rK71J"
      },
      "source": [
        "## Utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "1QuSTxwCINC9"
      },
      "outputs": [],
      "source": [
        "def get_num_words_per_sample(sample_texts):\n",
        "    \"\"\"Gets the median number of words per sample given corpus.\n",
        "\n",
        "    # Arguments\n",
        "        sample_texts: list, sample texts.\n",
        "\n",
        "    # Returns\n",
        "        int, median number of words per sample.\n",
        "    \"\"\"\n",
        "    num_words = [len(s.split()) for s in sample_texts]\n",
        "    return np.median(num_words)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xxP6y3tQK_IF"
      },
      "source": [
        "## Key metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D_xhI3z6Gw1J",
        "outputId": "093b942b-f7a7-4388-984e-070f912c137f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Xtrain key metrics\n",
            "Number of samples: 1688\n",
            "Number of aspects: 9\n",
            "Number of classes: 27\n",
            "Number of words per sample: 16.0\n",
            "Number of samples/number of words per sample ratio 105.5\n"
          ]
        }
      ],
      "source": [
        "num_samples = len(xtrain)\n",
        "num_aspects = len(aspects)\n",
        "num_classes = num_aspects * 3\n",
        "num_words_per_sample = get_num_words_per_sample(xtrain)\n",
        "sw_ratio = num_samples / num_words_per_sample\n",
        "\n",
        "\n",
        "print(\"Xtrain key metrics\")\n",
        "print(\"Number of samples:\", num_samples)\n",
        "print(\"Number of aspects:\", num_aspects)\n",
        "print(\"Number of classes:\", num_classes)\n",
        "print(\"Number of words per sample:\", num_words_per_sample)\n",
        "print(\"Number of samples/number of words per sample ratio\", sw_ratio)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "03IBx4ZS8tn4"
      },
      "source": [
        "## Class distribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "ohAmNN9b50AH",
        "outputId": "d1b0459b-3732-4ffe-f652-a4198e9e2b24"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"203a656d-1707-4be8-8755-375655c3fe56\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"203a656d-1707-4be8-8755-375655c3fe56\")) {                    Plotly.newPlot(                        \"203a656d-1707-4be8-8755-375655c3fe56\",                        [{\"labels\":[\"HOTEL_ROOM#QUALITY\",\"HOTEL_ROOM#PRICES\",\"HOTEL_ROOM#CLEANLINESS\",\"HOTEL_ROOM#GENERAL\",\"FACILITY#GENERAL\",\"FOOD#QUALITY\",\"FOOD#OPTION\",\"SERVICE#GENERAL\",\"LOCATION#GENERAL\"],\"name\":\"Train\",\"textposition\":\"inside\",\"values\":[1971,564,1846,925,1241,878,329,2864,1297],\"type\":\"pie\",\"domain\":{\"x\":[0.0,0.2888888888888889],\"y\":[0.0,1.0]}},{\"labels\":[\"HOTEL_ROOM#QUALITY\",\"HOTEL_ROOM#PRICES\",\"HOTEL_ROOM#CLEANLINESS\",\"HOTEL_ROOM#GENERAL\",\"FACILITY#GENERAL\",\"FOOD#QUALITY\",\"FOOD#OPTION\",\"SERVICE#GENERAL\",\"LOCATION#GENERAL\"],\"name\":\"Dev\",\"textposition\":\"inside\",\"values\":[276,112,228,189,141,71,34,409,240],\"type\":\"pie\",\"domain\":{\"x\":[0.35555555555555557,0.6444444444444445],\"y\":[0.0,1.0]}},{\"labels\":[\"HOTEL_ROOM#QUALITY\",\"HOTEL_ROOM#PRICES\",\"HOTEL_ROOM#CLEANLINESS\",\"HOTEL_ROOM#GENERAL\",\"FACILITY#GENERAL\",\"FOOD#QUALITY\",\"FOOD#OPTION\",\"SERVICE#GENERAL\",\"LOCATION#GENERAL\"],\"name\":\"Test\",\"textposition\":\"inside\",\"values\":[490,177,504,274,356,189,58,773,424],\"type\":\"pie\",\"domain\":{\"x\":[0.7111111111111111,1.0],\"y\":[0.0,1.0]}}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"annotations\":[{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Train\",\"x\":0.14444444444444446,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Dev\",\"x\":0.5,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Test\",\"x\":0.8555555555555556,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"title\":{\"text\":\"% of samples per aspect\"}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('203a656d-1707-4be8-8755-375655c3fe56');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import plotly.graph_objects as go\n",
        "from plotly.subplots import make_subplots\n",
        "\n",
        "def get_pie(df, name):\n",
        "    count = df.sum(axis=0)\n",
        "    return go.Pie(labels=count.index, values=count,\n",
        "                  textposition='inside', name=name)\n",
        "\n",
        "\n",
        "names = ('Train', 'Dev', 'Test')\n",
        "phaseA = (ytrain, ydev, ytest)\n",
        "\n",
        "fig = make_subplots(cols=3, subplot_titles=names,\n",
        "                    specs=[[{'type': 'pie'}] * 3])\n",
        "\n",
        "for i, (df, name) in enumerate(zip(phaseA, names), 1):\n",
        "    fig.add_trace(get_pie(df, name), row=1, col=i)\n",
        "\n",
        "fig.update_layout(title='% of samples per aspect')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "VQLRJkhS1t83",
        "outputId": "944c6be3-fadf-48d6-f40d-9fa36898ee8c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"be141bd0-89e5-4563-a749-0ad5aba4894b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"be141bd0-89e5-4563-a749-0ad5aba4894b\")) {                    Plotly.newPlot(                        \"be141bd0-89e5-4563-a749-0ad5aba4894b\",                        [{\"labels\":[\"HOTEL_ROOM#QUALITY -\",\"HOTEL_ROOM#QUALITY o\",\"HOTEL_ROOM#QUALITY +\",\"HOTEL_ROOM#PRICES -\",\"HOTEL_ROOM#PRICES o\",\"HOTEL_ROOM#PRICES +\",\"HOTEL_ROOM#CLEANLINESS -\",\"HOTEL_ROOM#CLEANLINESS o\",\"HOTEL_ROOM#CLEANLINESS +\",\"HOTEL_ROOM#GENERAL -\",\"HOTEL_ROOM#GENERAL o\",\"HOTEL_ROOM#GENERAL +\",\"FACILITY#GENERAL -\",\"FACILITY#GENERAL o\",\"FACILITY#GENERAL +\",\"FOOD#QUALITY -\",\"FOOD#QUALITY o\",\"FOOD#QUALITY +\",\"FOOD#OPTION -\",\"FOOD#OPTION o\",\"FOOD#OPTION +\",\"SERVICE#GENERAL -\",\"SERVICE#GENERAL o\",\"SERVICE#GENERAL +\",\"LOCATION#GENERAL -\",\"LOCATION#GENERAL o\",\"LOCATION#GENERAL +\"],\"name\":\"Train\",\"textposition\":\"inside\",\"values\":[192,51,559,22,16,170,162,5,558,54,56,253,289,53,282,74,72,220,83,3,80,225,40,853,29,34,400],\"type\":\"pie\",\"domain\":{\"x\":[0.0,0.2888888888888889],\"y\":[0.0,1.0]}},{\"labels\":[\"HOTEL_ROOM#QUALITY -\",\"HOTEL_ROOM#QUALITY o\",\"HOTEL_ROOM#QUALITY +\",\"HOTEL_ROOM#PRICES -\",\"HOTEL_ROOM#PRICES o\",\"HOTEL_ROOM#PRICES +\",\"HOTEL_ROOM#CLEANLINESS -\",\"HOTEL_ROOM#CLEANLINESS o\",\"HOTEL_ROOM#CLEANLINESS +\",\"HOTEL_ROOM#GENERAL -\",\"HOTEL_ROOM#GENERAL o\",\"HOTEL_ROOM#GENERAL +\",\"FACILITY#GENERAL -\",\"FACILITY#GENERAL o\",\"FACILITY#GENERAL +\",\"FOOD#QUALITY -\",\"FOOD#QUALITY o\",\"FOOD#QUALITY +\",\"FOOD#OPTION -\",\"FOOD#OPTION o\",\"FOOD#OPTION +\",\"SERVICE#GENERAL -\",\"SERVICE#GENERAL o\",\"SERVICE#GENERAL +\",\"LOCATION#GENERAL -\",\"LOCATION#GENERAL o\",\"LOCATION#GENERAL +\"],\"name\":\"Dev\",\"textposition\":\"inside\",\"values\":[31,4,79,9,2,33,26,2,66,13,7,54,52,4,27,10,5,17,11,1,7,41,7,118,5,2,77],\"type\":\"pie\",\"domain\":{\"x\":[0.35555555555555557,0.6444444444444445],\"y\":[0.0,1.0]}},{\"labels\":[\"HOTEL_ROOM#QUALITY -\",\"HOTEL_ROOM#QUALITY o\",\"HOTEL_ROOM#QUALITY +\",\"HOTEL_ROOM#PRICES -\",\"HOTEL_ROOM#PRICES o\",\"HOTEL_ROOM#PRICES +\",\"HOTEL_ROOM#CLEANLINESS -\",\"HOTEL_ROOM#CLEANLINESS o\",\"HOTEL_ROOM#CLEANLINESS +\",\"HOTEL_ROOM#GENERAL -\",\"HOTEL_ROOM#GENERAL o\",\"HOTEL_ROOM#GENERAL +\",\"FACILITY#GENERAL -\",\"FACILITY#GENERAL o\",\"FACILITY#GENERAL +\",\"FOOD#QUALITY -\",\"FOOD#QUALITY o\",\"FOOD#QUALITY +\",\"FOOD#OPTION -\",\"FOOD#OPTION o\",\"FOOD#OPTION +\",\"SERVICE#GENERAL -\",\"SERVICE#GENERAL o\",\"SERVICE#GENERAL +\",\"LOCATION#GENERAL -\",\"LOCATION#GENERAL o\",\"LOCATION#GENERAL +\"],\"name\":\"Test\",\"textposition\":\"inside\",\"values\":[66,8,136,20,5,49,55,7,145,28,15,72,106,14,74,20,17,45,18,2,12,82,14,221,10,12,130],\"type\":\"pie\",\"domain\":{\"x\":[0.7111111111111111,1.0],\"y\":[0.0,1.0]}}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"annotations\":[{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Train\",\"x\":0.14444444444444446,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Dev\",\"x\":0.5,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"},{\"font\":{\"size\":16},\"showarrow\":false,\"text\":\"Test\",\"x\":0.8555555555555556,\"xanchor\":\"center\",\"xref\":\"paper\",\"y\":1.0,\"yanchor\":\"bottom\",\"yref\":\"paper\"}],\"title\":{\"text\":\"% of samples per class (entity, sentiment)\"}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('be141bd0-89e5-4563-a749-0ad5aba4894b');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "phaseML = (ytrain_ml, ydev_ml, ytest_ml)\n",
        "\n",
        "fig = make_subplots(cols=3, subplot_titles=names,\n",
        "                    specs=[[{'type': 'pie'}] * 3])\n",
        "\n",
        "for i, (df, name) in enumerate(zip(phaseML, names), 1):\n",
        "    fig.add_trace(get_pie(df, name), row=1, col=i)\n",
        "\n",
        "fig.update_layout(title='% of samples per class (entity, sentiment)')"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "sZ9lbnGt9Kd6"
      },
      "source": [
        "## Sample length distribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "JHRo6N3cjlbm",
        "outputId": "4581c2be-b5a9-45f6-d974-b1703ace2d3c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"eb8577cf-3f1a-4670-85d2-78ff05510244\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"eb8577cf-3f1a-4670-85d2-78ff05510244\")) {                    Plotly.newPlot(                        \"eb8577cf-3f1a-4670-85d2-78ff05510244\",                        [{\"name\":\"train\",\"x\":[159,96,201,78,64,45,69,139,275,84,85,72,62,481,72,67,202,52,62,60,419,123,268,61,135,61,173,108,408,30,43,45,49,464,86,48,48,40,48,42,57,448,88,80,169,368,113,188,63,143,42,49,146,178,47,78,60,65,195,71,69,134,62,80,48,65,62,133,52,45,38,133,65,34,58,52,65,145,136,145,54,549,34,223,67,56,274,113,76,711,30,153,31,85,106,36,70,37,70,54,73,63,458,79,231,301,582,79,48,109,90,166,70,28,39,120,64,130,121,347,111,513,120,162,163,206,104,108,143,82,59,53,210,38,42,72,75,52,253,30,76,52,170,98,35,121,55,39,937,86,124,206,196,340,49,150,39,111,64,651,47,53,169,30,390,176,116,56,86,195,203,558,48,28,59,433,88,30,205,43,46,394,107,120,328,224,59,76,95,540,298,148,283,32,213,33,73,72,156,272,713,39,46,573,265,133,41,53,143,213,48,972,135,111,75,43,676,228,41,145,56,125,125,29,97,141,46,124,59,55,30,89,254,89,66,51,72,75,250,62,491,61,65,214,29,190,41,75,296,69,609,92,58,42,50,92,42,396,122,127,107,451,271,199,41,55,98,198,79,49,210,161,156,237,123,82,335,220,160,374,32,473,41,214,91,86,220,104,72,47,71,253,419,171,41,37,56,51,69,270,38,97,39,153,30,190,31,38,264,94,379,285,549,56,173,245,285,67,85,279,74,407,128,210,471,112,91,107,189,142,30,220,51,68,57,199,39,39,35,59,30,265,33,268,53,38,39,896,637,448,77,146,209,31,56,48,56,84,60,60,153,215,58,92,38,106,413,160,29,31,64,157,117,77,50,46,42,89,67,100,28,194,39,57,26,45,26,84,155,90,650,54,133,278,38,88,98,161,47,270,62,31,444,184,45,30,189,186,47,56,65,43,295,127,338,98,183,62,34,139,146,1078,169,45,160,205,234,47,115,33,221,124,165,172,228,254,138,221,43,117,158,582,62,386,127,58,613,551,41,220,289,51,74,77,123,116,71,95,219,30,98,28,204,122,224,78,322,86,92,251,544,263,56,130,167,65,273,36,95,129,116,188,167,244,33,34,129,163,121,72,53,101,102,180,92,206,59,81,57,61,47,86,34,187,57,71,136,139,102,14,234,89,73,176,109,116,44,346,404,145,273,120,70,70,422,86,66,204,312,425,79,259,266,157,67,42,29,49,89,32,39,86,53,84,73,82,244,155,84,195,227,38,38,321,31,118,392,95,31,150,73,70,58,222,78,80,90,58,124,187,219,280,288,316,45,62,71,112,92,110,78,73,204,71,240,70,118,1097,158,202,64,82,153,384,238,83,121,31,75,284,180,73,38,62,67,578,40,48,162,237,50,61,87,288,109,149,133,100,56,438,65,135,27,177,275,73,94,212,82,82,195,47,208,297,73,142,77,78,121,68,40,89,41,58,303,43,454,41,187,168,150,104,203,408,124,187,53,91,194,79,42,201,394,167,72,97,35,167,166,189,304,52,129,147,604,284,52,43,306,98,35,149,49,259,32,36,62,103,58,22,1100,148,284,51,67,369,119,405,68,37,50,80,67,40,60,62,40,123,378,186,123,44,70,121,41,85,32,48,93,666,135,388,55,47,41,142,265,142,48,108,255,149,104,118,370,74,40,120,156,309,80,269,92,252,70,126,70,55,73,33,66,35,65,30,98,89,40,78,124,86,206,134,174,63,167,113,279,54,485,36,41,41,41,55,202,312,53,97,78,53,140,103,41,107,48,625,297,238,165,249,50,76,85,104,73,171,38,42,35,84,27,38,53,167,129,29,56,45,41,32,48,73,62,255,30,40,33,36,36,40,48,74,61,35,43,50,39,34,863,33,43,40,54,64,29,36,31,121,43,36,37,61,35,29,41,296,424,106,53,50,113,61,184,57,37,42,40,103,62,76,37,70,439,44,41,35,98,111,40,100,68,25,39,122,39,77,45,81,40,289,26,100,30,108,59,25,37,29,84,130,72,70,151,143,32,35,41,228,102,31,47,33,37,54,27,48,308,33,45,78,258,50,43,123,70,101,167,190,324,95,343,34,32,94,43,236,37,203,271,32,25,156,79,276,41,86,187,92,57,63,90,58,31,152,35,124,164,83,56,60,29,103,196,104,220,274,31,86,29,43,41,32,322,48,52,51,112,74,232,111,164,86,34,64,205,167,69,76,34,161,53,69,1009,42,44,41,112,55,103,300,75,225,134,131,30,274,46,63,95,33,41,113,68,149,119,498,490,44,179,95,53,101,148,57,116,318,841,236,86,72,92,162,396,289,145,151,78,44,597,325,71,102,56,54,908,328,350,88,161,228,286,200,128,86,859,121,129,233,371,137,234,70,289,125,53,42,80,51,205,82,94,90,375,512,91,103,59,188,107,255,75,130,42,176,109,170,67,140,704,702,51,220,28,38,23,76,311,136,102,371,62,94,82,30,501,58,204,87,65,73,60,97,385,78,86,153,62,32,146,77,124,36,53,220,39,42,76,48,89,69,107,121,159,91,241,49,80,97,85,65,145,95,104,401,81,57,66,255,43,252,95,128,181,89,189,269,164,67,34,124,184,27,242,427,64,193,241,175,36,64,188,140,122,28,86,315,245,322,116,410,397,209,54,328,672,191,560,257,136,111,131,126,127,100,209,343,62,196,54,32,101,80,192,105,524,293,49,334,376,174,192,52,254,38,337,353,167,286,342,61,174,52,257,187,100,147,60,61,31,160,306,209,75,63,73,172,179,68,96,111,470,79,55,80,92,44,63,98,287,98,231,92,60,54,282,35,297,49,34,60,110,68,66,105,202,222,41,71,67,193,325,272,28,235,96,38,63,188,94,121,50,44,101,193,28,112,93,165,83,31,244,81,148,96,36,69,140,67,54,38,235,155,121,226,62,32,178,149,194,312,146,35,349,45,61,34,34,159,48,233,147,196,52,44,268,62,97,60,293,116,57,44,38,147,55,57,55,76,311,543,86,51,291,128,37,147,47,48,236,186,52,75,49,43,92,41,207,45,110,536,71,29,212,476,163,83,306,38,42,121,100,201,362,48,886,207,34,71,275,34,199,76,40,364,37,96,1012,85,84,53,51,79,203,49,31,28,34,751,69,58,47,126,279,106,77,113,45,436,38,190,953,63,219,196,31,259,86,294,31,79,97,167,57,46,40,123,262,90,43,84,87,159,288,110,425,90,28,77,77,55,184,316,309,229,37,225,181,244,154,451,280,141,119,141,119,183,55,121,32,320,506,86,215,178,176,231,120,96,62,51,160,54,60,46,36,507,77,53,67,75,51,135,302,91,99,74,101,407,89,112,291,80,64,277,365,45,141,94,135,42,74,63,250,217,121,120,269,100,145,48,299,58,353,89,59,34,42,41,83,115,30,77,57,42,53,66,38,30,39,35,62,92,68,79,62,31,116,174,34,46,59,73,53,85,36,104,120,241,30,330,60,121,462,39,76,75,37,103,276,687,32,182,32,175,206,143,323,115,48,76,160,222,97,123,48,133,58,54,41,73,218,49,111,50,50,50,179,39,108,101,159,59,10,43,87,73,174,33,92,147,105,350,74,159,160,42,349,248,148,166,51,140,195,38,238,201,174,105,205,59,170,223,61,42,298,54,62,34,94,58,126,38,480,84,160,153,47,181,169,38,217,49,108,37,55,103,178,100,65,142,227,332,66,153,254,142,79,30,44,92,455,411,123,47,74,82,134,172,163,31,80,103,107,85,53,73,46,51,225,275,209,79,97,69,184,63,62,316,42,68,30,364,45,257,71,79,175,38],\"type\":\"histogram\",\"opacity\":0.5},{\"name\":\"dev\",\"x\":[74,85,387,552,150,80,41,50,47,237,69,328,297,72,158,206,53,62,42,243,72,101,38,42,114,69,110,148,39,334,86,39,31,58,56,55,216,120,112,83,41,83,213,78,137,115,35,56,186,38,45,54,199,40,30,32,76,123,231,45,32,97,143,446,31,50,319,327,74,184,110,260,53,87,90,270,97,168,85,34,240,37,30,334,79,65,32,303,60,54,72,70,92,63,50,59,175,214,66,78,136,52,76,589,65,141,82,64,217,163,48,345,44,49,360,120,141,46,39,67,100,121,58,193,273,441,36,132,54,39,61,68,45,265,308,111,66,103,87,235,43,85,115,63,61,211,86,141,185,442,179,346,150,120,262,325,454,104,163,165,126,30,83,252,127,187,119,33,51,119,209,35,36,79,332,34,127,206,46,66,212,33,108,73,167,53,330,154,104,50,59,54,96,48,76,151,172,162,52,170,88,428,139,188,32,628,158,90,284,704,845,555,187,45,125,229,205,177,36,706,58,43,66,31,212,70,279,159,59,275,76,164,237,95,44,346,117,120,130,61,236,25,158,95,99,93,110,242,660,52,77,332],\"type\":\"histogram\",\"opacity\":0.5},{\"name\":\"test\",\"x\":[125,107,127,59,618,67,127,113,157,118,126,220,92,183,42,86,347,44,73,80,234,234,343,400,86,275,44,62,384,57,287,63,146,62,96,1072,461,102,58,286,66,37,92,47,58,56,67,70,149,58,63,104,90,45,47,157,44,83,37,72,45,182,31,36,254,133,332,97,61,183,82,139,35,89,45,78,35,166,33,30,189,57,44,438,97,137,82,100,102,48,193,56,92,64,59,126,40,60,97,41,46,159,61,30,113,283,39,768,84,47,276,124,38,65,92,110,42,74,88,36,52,146,82,113,39,48,188,130,97,140,128,131,124,257,125,221,389,211,472,244,127,44,391,71,161,167,73,122,172,338,291,342,248,147,130,81,73,38,47,48,323,468,89,90,126,206,166,153,138,363,86,251,288,444,57,101,34,143,68,124,146,43,89,142,53,112,90,93,40,147,97,187,366,48,129,98,780,44,54,161,93,56,109,163,397,106,73,32,36,605,31,38,356,58,124,196,57,75,54,53,78,39,226,126,59,77,93,119,235,55,34,88,58,957,164,185,31,98,42,266,62,290,79,35,77,106,40,153,259,77,485,178,46,73,77,37,455,35,62,109,38,97,61,37,168,249,48,117,194,235,53,352,81,73,105,183,29,348,91,100,177,56,83,31,92,70,79,81,49,152,84,233,36,90,395,238,54,165,86,391,46,201,72,320,330,62,66,47,39,139,54,108,103,130,554,64,119,52,143,102,43,413,131,76,116,118,136,41,364,56,130,457,53,101,73,124,38,107,83,166,87,53,161,113,47,686,441,83,186,150,72,90,32,159,255,212,194,29,52,406,32,162,58,51,95,56,347,546,121,85,55,268,223,109,48,143,47,63,192,177,106,52,30,91,106,234,39,39,24,85,66,70,35,80,131,69,78,44,194,77,86,198,37,32,150,56,52,205,127,96,64,43,42,30,32,67,49,64,86,30,41,35,254,29,131,66,79,64,139,31,80,228,61,115,92,40,50,148,250,439,195,29,74,58,32,415,185,230,61,81,83,32,43,105,209,106,65,79,89,33,295,194,76,179,40,82,159,73,103,34,58,45,162,205,59,372,63,75,68,103,54,792,52,115,280,231,60,92,836,33,165,322,232,197,704,738,172,145,109,206,215,90,151,102,244,44,79],\"type\":\"histogram\",\"opacity\":0.5}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"title\":{\"text\":\"Sample length distribution\"},\"barmode\":\"overlay\"},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('eb8577cf-3f1a-4670-85d2-78ff05510244');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "count0 = [len(s) for s in xtrain]\n",
        "count1 = [len(s) for s in xdev]\n",
        "count2 = [len(s) for s in xtest]\n",
        "\n",
        "fig = go.Figure()\n",
        "fig.add_trace(go.Histogram(x=count0, name='train'))\n",
        "fig.add_trace(go.Histogram(x=count1, name='dev'))\n",
        "fig.add_trace(go.Histogram(x=count2, name='test'))\n",
        "\n",
        "fig.update_layout(title='Sample length distribution', barmode='overlay')\n",
        "fig.update_traces(opacity=0.5)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "F0sd4C9fO739"
      },
      "source": [
        "#Out of Vocab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iIUVrfpiPBsv",
        "outputId": "27d6b868-8293-4809-bcb7-3c454d559fe0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0       phòng giá rẻ đến bất_ngờ mình ở phòng senior n...\n",
              "1       tiếp_tân dễ_thương nhiệt_tình _phòng có view r...\n",
              "2       chìa khóa phòng bị lỗi nhét thẻ vào khay rồi n...\n",
              "3       quá ổn so với giá tiền gần biển tiện_ích xung_...\n",
              "4       nhân_viên khách_sạn rất nhiệt_tình chắc_chắn s...\n",
              "                              ...                        \n",
              "1683    thật vui khi gia_đình mình ở khách_sạn wzjfj12...\n",
              "1684    khách_sạn mới tốt về mọi mặt lễ_tân nhiệt_tình...\n",
              "1685    không_gian sáng chăm_sóc khách_hàng tuyệt_vời ...\n",
              "1686    nhân_viên thân_thiện vị_trí cực_kỳ_trung_tâm v...\n",
              "1687               phòng đẹp và sạch_sẽ chắc_chắn đề_nghị\n",
              "Name: review, Length: 1688, dtype: object"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "xtrain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "mQNgeFaJO7X4"
      },
      "outputs": [],
      "source": [
        "train_vocabulary = set()\n",
        "for line in xtrain:\n",
        "    words = line.strip().split()\n",
        "    train_vocabulary.update(words)\n",
        "\n",
        "# Tạo từ vựng của tập test\n",
        "test_vocabulary = set()\n",
        "for line in xtest:\n",
        "    words = line.strip().split()\n",
        "    test_vocabulary.update(words)\n",
        "\n",
        "# Tạo từ vựng của tập val\n",
        "val_vocabulary = set()\n",
        "for line in xdev:\n",
        "    words = line.strip().split()\n",
        "    val_vocabulary.update(words)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hEpywaQ0PPIN",
        "outputId": "dd931fc2-dbf4-4614-c20b-79e5d2ddfa8a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Số lượng từ vựng tập train:  3382\n",
            "Số lượng từ vựng tập val:  1303\n",
            "Số lượng từ vựng tập test:  1879\n",
            "Số lượng từ vựng nằm ngoài tập train của tập val:  234\n",
            "Số lượng từ vựng nằm ngoài tập train của tập test:  466\n"
          ]
        }
      ],
      "source": [
        "print(\"Số lượng từ vựng tập train: \", len(train_vocabulary))\n",
        "print(\"Số lượng từ vựng tập val: \", len(val_vocabulary))\n",
        "print(\"Số lượng từ vựng tập test: \", len(test_vocabulary))\n",
        "\n",
        "print(\"Số lượng từ vựng nằm ngoài tập train của tập val: \", len(val_vocabulary - train_vocabulary))\n",
        "print(\"Số lượng từ vựng nằm ngoài tập train của tập test: \", len(test_vocabulary - train_vocabulary))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "tz4aAURfgKIW"
      },
      "source": [
        "# Feature extraction (Convert reviews to vectors)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "v2UquVpDgbvH"
      },
      "source": [
        "##  Basic features (1, 2, 3 grams)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YTpdlwwCgY2o"
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "vectorizer = TfidfVectorizer(ngram_range=(1, 3),\n",
        "                             min_df=2, max_df=0.9)\n",
        "\n",
        "# x data using basic clean up class and basic features extrator\n",
        "xtrain_tfidf = vectorizer.fit_transform(xtrain)\n",
        "xdev_tfidf   = vectorizer.transform(xdev)\n",
        "xtest_tfidf  = vectorizer.transform(xtest)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "egBRGMSHncKX",
        "outputId": "8105a4c7-ecfe-4b2b-8fe1-0942a7562256"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(1688, 9237)"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "xtrain_tfidf.shape"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "cell_id": "00008-3e35d1d6-b9cd-4dc2-a45e-ccf833534cd9",
        "deepnote_cell_type": "markdown",
        "id": "ppKYl2LZPra0",
        "tags": []
      },
      "source": [
        "# Model Architectures"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "rFrqY_VnqN-2"
      },
      "source": [
        "## End-to-End Architecture"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hpwu1HJLqWmT"
      },
      "outputs": [],
      "source": [
        "from sklearn.multioutput import MultiOutputClassifier as MOC"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "cell_id": "00011-0d327c11-f739-49f4-b198-560e8b9d6e52",
        "deepnote_cell_type": "markdown",
        "id": "J7cEYevzPrbB",
        "tags": []
      },
      "source": [
        "# Evaluation functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UWRymi7IuI7M",
        "outputId": "8c374ab0-b37d-4dff-a153-b179d7ecdfff"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ],
      "source": [
        "#!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RJ952C7RUM6Y"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score, classification_report\n",
        "from eval import label_map, replacements, target_names, aspect_detection_eval, sentiment_classification_eval, combination_eval\n",
        "\n",
        "categories= list(ytest.columns)\n",
        "\n",
        "def quick_f1(y_true, y_pred):\n",
        "    y_pred = mo2ml(mo2df(y_pred))\n",
        "    return round(f1_score(y_true, y_pred, average='micro', zero_division=0), 4)\n",
        "\n",
        "def evaluate(model, X, y, average='micro'):\n",
        "    yb_true  = mo2ml(y)\n",
        "\n",
        "    yb_pred  = mo2df(model.predict(X))\n",
        "    yb_pred  = mo2ml(yb_pred)\n",
        "\n",
        "    return classification_report(yb_true, yb_pred, zero_division=0)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "1mOvkWcI-j4u"
      },
      "source": [
        "# Compare models"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "6quUiO0590xf"
      },
      "source": [
        "## Base Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AS1uwzHL2FEq"
      },
      "outputs": [],
      "source": [
        "from sklearn.svm import LinearSVC\n",
        "from sklearn.linear_model import LogisticRegression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vL6MjQ6kmLwm"
      },
      "outputs": [],
      "source": [
        "clf0 = MOC(LinearSVC(random_state=5))\n",
        "clf0.fit(xtrain_tfidf, ytrain)\n",
        "ypred= clf0.predict(xtest_tfidf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dQlvCT30vSr4"
      },
      "outputs": [],
      "source": [
        "ypred= pd.DataFrame(ypred, columns= categories)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q-fISOS2v9dr",
        "outputId": "a60fc6c5-0270-4722-f99f-3e54ca2738d9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Aspect Detection Evaluate ##\n",
            "                        precision    recall  f1-score   support\n",
            "\n",
            "                           0.8766    0.9538    0.9136      3180\n",
            "      FACILITY#GENERAL     0.8387    0.6701    0.7450       194\n",
            "           FOOD#OPTION     0.8947    0.5312    0.6667        32\n",
            "          FOOD#QUALITY     0.8776    0.5244    0.6565        82\n",
            "HOTEL_ROOM#CLEANLINESS     0.9175    0.8599    0.8878       207\n",
            "    HOTEL_ROOM#GENERAL     0.7692    0.2609    0.3896       115\n",
            "     HOTEL_ROOM#PRICES     0.8125    0.5270    0.6393        74\n",
            "    HOTEL_ROOM#QUALITY     0.7784    0.7190    0.7475       210\n",
            "      LOCATION#GENERAL     0.9231    0.5526    0.6914       152\n",
            "       SERVICE#GENERAL     0.9045    0.8959    0.9002       317\n",
            "\n",
            "              accuracy                         0.8742      4563\n",
            "             macro avg     0.8593    0.6495    0.7237      4563\n",
            "          weighted avg     0.8722    0.8742    0.8652      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "aspect_detection_eval(ytest, ypred)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p7ew2UrdysbI",
        "outputId": "bbbe7c3f-b896-41cc-c49f-1e1d5c1170c4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Sentiment Classification Evaluate ##\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        None     0.8766    0.9538    0.9136      3180\n",
            "    negative     0.7177    0.5210    0.6037       405\n",
            "     neutral     0.4000    0.0426    0.0769        94\n",
            "    positive     0.7960    0.7195    0.7558       884\n",
            "\n",
            "    accuracy                         0.8512      4563\n",
            "   macro avg     0.6976    0.5592    0.5875      4563\n",
            "weighted avg     0.8371    0.8512    0.8383      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "sentiment_classification_eval(ytest, ypred)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KhF7DcqPwDAW",
        "outputId": "9c5e184a-fb57-4381-ca94-03df97fdda92"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Combination Evaluate (Aspect Detection + Sentiment Classification) ##\n",
            "                                 precision    recall  f1-score   support\n",
            "\n",
            "          FACILITY#GENERAL,None     0.8182    0.9201    0.8662       313\n",
            "      FACILITY#GENERAL,negative     0.7447    0.6604    0.7000       106\n",
            "       FACILITY#GENERAL,neutral     1.0000    0.0714    0.1333        14\n",
            "      FACILITY#GENERAL,positive     0.7500    0.6081    0.6716        74\n",
            "               FOOD#OPTION,None     0.9693    0.9958    0.9823       475\n",
            "           FOOD#OPTION,negative     0.7273    0.4444    0.5517        18\n",
            "            FOOD#OPTION,neutral     1.0000    0.0000    0.0000         2\n",
            "           FOOD#OPTION,positive     0.8750    0.5833    0.7000        12\n",
            "              FOOD#QUALITY,None     0.9148    0.9859    0.9490       425\n",
            "          FOOD#QUALITY,negative     0.6923    0.4500    0.5455        20\n",
            "           FOOD#QUALITY,neutral     0.5000    0.1176    0.1905        17\n",
            "          FOOD#QUALITY,positive     0.8438    0.6000    0.7013        45\n",
            "    HOTEL_ROOM#CLEANLINESS,None     0.9073    0.9467    0.9266       300\n",
            "HOTEL_ROOM#CLEANLINESS,negative     0.8000    0.5818    0.6737        55\n",
            " HOTEL_ROOM#CLEANLINESS,neutral     1.0000    0.0000    0.0000         7\n",
            "HOTEL_ROOM#CLEANLINESS,positive     0.8831    0.9379    0.9097       145\n",
            "        HOTEL_ROOM#GENERAL,None     0.8184    0.9770    0.8907       392\n",
            "    HOTEL_ROOM#GENERAL,negative     0.6667    0.0714    0.1290        28\n",
            "     HOTEL_ROOM#GENERAL,neutral     0.5000    0.0667    0.1176        15\n",
            "    HOTEL_ROOM#GENERAL,positive     0.6471    0.3056    0.4151        72\n",
            "         HOTEL_ROOM#PRICES,None     0.9237    0.9792    0.9507       433\n",
            "     HOTEL_ROOM#PRICES,negative     1.0000    0.0000    0.0000        20\n",
            "      HOTEL_ROOM#PRICES,neutral     1.0000    0.0000    0.0000         5\n",
            "     HOTEL_ROOM#PRICES,positive     0.6042    0.5918    0.5979        49\n",
            "        HOTEL_ROOM#QUALITY,None     0.8115    0.8552    0.8328       297\n",
            "    HOTEL_ROOM#QUALITY,negative     0.6667    0.5455    0.6000        66\n",
            "     HOTEL_ROOM#QUALITY,neutral     0.0000    0.0000    0.0000         8\n",
            "    HOTEL_ROOM#QUALITY,positive     0.7299    0.7353    0.7326       136\n",
            "          LOCATION#GENERAL,None     0.8365    0.9803    0.9027       355\n",
            "      LOCATION#GENERAL,negative     1.0000    0.0000    0.0000        10\n",
            "       LOCATION#GENERAL,neutral     1.0000    0.0000    0.0000        12\n",
            "      LOCATION#GENERAL,positive     0.8022    0.5615    0.6606       130\n",
            "           SERVICE#GENERAL,None     0.8290    0.8421    0.8355       190\n",
            "       SERVICE#GENERAL,negative     0.6835    0.6585    0.6708        82\n",
            "        SERVICE#GENERAL,neutral     1.0000    0.0000    0.0000        14\n",
            "       SERVICE#GENERAL,positive     0.8383    0.8914    0.8640       221\n",
            "\n",
            "                       accuracy                         0.8512      4563\n",
            "                      macro avg     0.7995    0.4990    0.5195      4563\n",
            "                   weighted avg     0.8458    0.8512    0.8330      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "combination_eval(ytest, ypred)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "A-QGBUgCLzPE"
      },
      "source": [
        "## Linear SVC hyperparameter tuning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NAh9wdhKpp1h"
      },
      "outputs": [],
      "source": [
        "import optuna\n",
        "from optuna.samplers import TPESampler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XwgrmIMsHkUq"
      },
      "outputs": [],
      "source": [
        "def callback(study, trial):\n",
        "    if study.best_trial.number == trial.number:\n",
        "        study.set_user_attr(key='best_model', value=trial.user_attrs['model'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4EHONgBRprGE",
        "outputId": "375b9758-c110-4aab-b251-fbeac3f23093"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[I 2023-06-30 03:14:11,132] A new study created in memory with name: no-name-628b8b41-bbc0-4baf-a3f1-d9b1b4945d47\n",
            "[I 2023-06-30 03:14:11,213] Trial 0 finished with value: 0.2453 and parameters: {'C': 1.9636582699290402e-07, 'class_weight': 'balanced', 'loss': 'hinge'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:14:11,328] Trial 1 finished with value: 0.2453 and parameters: {'C': 5.339536586472381e-06, 'class_weight': None, 'loss': 'squared_hinge'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:14:11,405] Trial 2 finished with value: 0.2453 and parameters: {'C': 1.3055563380836963e-09, 'class_weight': None, 'loss': 'hinge'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:14:11,483] Trial 3 finished with value: 0.2453 and parameters: {'C': 1.1682869614143264e-09, 'class_weight': None, 'loss': 'hinge'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:14:11,830] Trial 4 finished with value: 0.639 and parameters: {'C': 0.2804917948703948, 'class_weight': 'balanced', 'loss': 'hinge'}. Best is trial 4 with value: 0.639.\n",
            "[I 2023-06-30 03:14:11,980] Trial 5 finished with value: 0.2453 and parameters: {'C': 0.002674151911363306, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 4 with value: 0.639.\n",
            "[I 2023-06-30 03:14:12,160] Trial 6 finished with value: 0.5847 and parameters: {'C': 0.02828254640470946, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 4 with value: 0.639.\n",
            "[I 2023-06-30 03:14:12,404] Trial 7 finished with value: 0.6904 and parameters: {'C': 0.15640579894830894, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 7 with value: 0.6904.\n",
            "[I 2023-06-30 03:14:12,488] Trial 8 finished with value: 0.2453 and parameters: {'C': 2.6540634283127564e-06, 'class_weight': None, 'loss': 'hinge'}. Best is trial 7 with value: 0.6904.\n",
            "[I 2023-06-30 03:14:12,688] Trial 9 finished with value: 0.59 and parameters: {'C': 0.03412839819879307, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 7 with value: 0.6904.\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "[I 2023-06-30 03:14:16,890] Trial 10 finished with value: 0.6875 and parameters: {'C': 78.77092791878178, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 7 with value: 0.6904.\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "[I 2023-06-30 03:14:20,781] Trial 11 finished with value: 0.6835 and parameters: {'C': 95.87563556023869, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 7 with value: 0.6904.\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "[I 2023-06-30 03:14:24,652] Trial 12 finished with value: 0.6861 and parameters: {'C': 81.85551797112431, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 7 with value: 0.6904.\n",
            "[I 2023-06-30 03:14:25,071] Trial 13 finished with value: 0.7204 and parameters: {'C': 1.3210904933837293, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:25,417] Trial 14 finished with value: 0.717 and parameters: {'C': 0.8670221819070405, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:25,912] Trial 15 finished with value: 0.7164 and parameters: {'C': 1.976960111199604, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:26,432] Trial 16 finished with value: 0.7183 and parameters: {'C': 2.0242166959094345, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:26,630] Trial 17 finished with value: 0.2453 and parameters: {'C': 0.0010659200839228614, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:27,519] Trial 18 finished with value: 0.7086 and parameters: {'C': 5.175181427700241, 'class_weight': None, 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:27,637] Trial 19 finished with value: 0.4468 and parameters: {'C': 0.01219024637212673, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:28,247] Trial 20 finished with value: 0.7126 and parameters: {'C': 5.468289950615913, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:28,535] Trial 21 finished with value: 0.7204 and parameters: {'C': 1.2943686678841066, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:28,722] Trial 22 finished with value: 0.7121 and parameters: {'C': 0.3025459846827313, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:29,441] Trial 23 finished with value: 0.7071 and parameters: {'C': 7.209060633923838, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:29,613] Trial 24 finished with value: 0.6934 and parameters: {'C': 0.17113522892361202, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:29,725] Trial 25 finished with value: 0.2966 and parameters: {'C': 0.005309165102722617, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:29,789] Trial 26 finished with value: 0.2453 and parameters: {'C': 0.00028579865158607386, 'class_weight': None, 'loss': 'hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:31,256] Trial 27 finished with value: 0.6981 and parameters: {'C': 20.60422431416515, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:31,534] Trial 28 finished with value: 0.7204 and parameters: {'C': 1.2557351302071564, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "[I 2023-06-30 03:14:32,017] Trial 29 finished with value: 0.7079 and parameters: {'C': 0.8265207842381129, 'class_weight': 'balanced', 'loss': 'hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:33,262] Trial 30 finished with value: 0.6985 and parameters: {'C': 15.682967374052172, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:33,521] Trial 31 finished with value: 0.7174 and parameters: {'C': 1.0491098725700234, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:33,661] Trial 32 finished with value: 0.6123 and parameters: {'C': 0.04274625115875294, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:34,007] Trial 33 finished with value: 0.7174 and parameters: {'C': 2.0187586786500713, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:35,511] Trial 34 finished with value: 0.6976 and parameters: {'C': 17.938811032114014, 'class_weight': None, 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:35,690] Trial 35 finished with value: 0.6182 and parameters: {'C': 0.21725524236200364, 'class_weight': 'balanced', 'loss': 'hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:35,821] Trial 36 finished with value: 0.5226 and parameters: {'C': 0.061483511453569256, 'class_weight': None, 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:36,567] Trial 37 finished with value: 0.7115 and parameters: {'C': 2.709653371234303, 'class_weight': 'balanced', 'loss': 'hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:36,795] Trial 38 finished with value: 0.717 and parameters: {'C': 0.7419721885834007, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:36,902] Trial 39 finished with value: 0.27 and parameters: {'C': 0.010719507271081527, 'class_weight': None, 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:37,173] Trial 40 finished with value: 0.6443 and parameters: {'C': 0.299702403854436, 'class_weight': 'balanced', 'loss': 'hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:37,433] Trial 41 finished with value: 0.7174 and parameters: {'C': 1.0431714743856146, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:37,651] Trial 42 finished with value: 0.6703 and parameters: {'C': 0.10058857753298989, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:39,050] Trial 43 finished with value: 0.7015 and parameters: {'C': 10.486525558559185, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "/usr/local/lib/python3.10/dist-packages/sklearn/svm/_base.py:1244: ConvergenceWarning:\n",
            "\n",
            "Liblinear failed to converge, increase the number of iterations.\n",
            "\n",
            "[I 2023-06-30 03:14:41,562] Trial 44 finished with value: 0.6951 and parameters: {'C': 30.128743215395602, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:41,975] Trial 45 finished with value: 0.7176 and parameters: {'C': 2.9695428511467394, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:42,481] Trial 46 finished with value: 0.7172 and parameters: {'C': 3.7739526862504995, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:44,224] Trial 47 finished with value: 0.6971 and parameters: {'C': 25.008486076565582, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:44,686] Trial 48 finished with value: 0.7181 and parameters: {'C': 3.364731908092535, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n",
            "[I 2023-06-30 03:14:44,883] Trial 49 finished with value: 0.7086 and parameters: {'C': 0.38168236121845145, 'class_weight': 'balanced', 'loss': 'squared_hinge'}. Best is trial 13 with value: 0.7204.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'C': 1.3210904933837293, 'class_weight': 'balanced', 'dual': True, 'fit_intercept': True, 'intercept_scaling': 1, 'loss': 'squared_hinge', 'max_iter': 2000, 'multi_class': 'ovr', 'penalty': 'l2', 'random_state': 5, 'tol': 0.0001, 'verbose': 0}\n",
            "{'C': 1.3210904933837293, 'class_weight': 'balanced', 'loss': 'squared_hinge'}\n"
          ]
        }
      ],
      "source": [
        "def linearsvc_objective(trial):\n",
        "    params = dict(\n",
        "        C=trial.suggest_float('C', 1e-9, 1e2, log=True),\n",
        "        class_weight=trial.suggest_categorical('class_weight', ['balanced', None]),\n",
        "        loss=trial.suggest_categorical('loss', ['hinge', 'squared_hinge']),\n",
        "        max_iter=2000,\n",
        "        random_state=5\n",
        "    )\n",
        "\n",
        "    clf = MOC(LinearSVC(**params))\n",
        "    clf.fit(xtrain_tfidf, ytrain)\n",
        "    trial.set_user_attr(key=\"model\", value=clf)\n",
        "\n",
        "    y_pred = clf.predict(xdev_tfidf)\n",
        "    return quick_f1(ydev_ml, y_pred)\n",
        "\n",
        "sampler = TPESampler(seed=22)\n",
        "linearsvc_study = optuna.create_study(sampler=sampler, direction='maximize')\n",
        "linearsvc_study.optimize(linearsvc_objective, n_trials=50, callbacks=[callback])\n",
        "\n",
        "\n",
        "clf2 = linearsvc_study.user_attrs['best_model']\n",
        "\n",
        "print(clf2.estimators_[0].get_params())\n",
        "print(linearsvc_study.best_params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2P14U1bIz63t"
      },
      "outputs": [],
      "source": [
        "ypred2=clf2.predict(xtest_tfidf)\n",
        "ypred2= pd.DataFrame(ypred2, columns= categories)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kw261T7g0bxt",
        "outputId": "18a67d2b-dc37-43f5-c964-8ab8bffc6e21"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Aspect Detection Evaluate ##\n",
            "                        precision    recall  f1-score   support\n",
            "\n",
            "                           0.8933    0.9377    0.9150      3180\n",
            "      FACILITY#GENERAL     0.8125    0.7371    0.7730       194\n",
            "           FOOD#OPTION     0.8696    0.6250    0.7273        32\n",
            "          FOOD#QUALITY     0.8281    0.6463    0.7260        82\n",
            "HOTEL_ROOM#CLEANLINESS     0.8857    0.8986    0.8921       207\n",
            "    HOTEL_ROOM#GENERAL     0.7037    0.3304    0.4497       115\n",
            "     HOTEL_ROOM#PRICES     0.8214    0.6216    0.7077        74\n",
            "    HOTEL_ROOM#QUALITY     0.7409    0.7762    0.7581       210\n",
            "      LOCATION#GENERAL     0.8679    0.6053    0.7132       152\n",
            "       SERVICE#GENERAL     0.9051    0.9022    0.9036       317\n",
            "\n",
            "              accuracy                         0.8786      4563\n",
            "             macro avg     0.8328    0.7080    0.7566      4563\n",
            "          weighted avg     0.8752    0.8786    0.8734      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "aspect_detection_eval(ytest, ypred2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CvEc97Qc0VKt",
        "outputId": "90f605bc-2f4f-40ef-c3bd-6e0d90ce41a7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Sentiment Classification Evaluate ##\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        None     0.8933    0.9377    0.9150      3180\n",
            "    negative     0.6472    0.5753    0.6092       405\n",
            "     neutral     0.4074    0.1170    0.1818        94\n",
            "    positive     0.7900    0.7489    0.7689       884\n",
            "\n",
            "    accuracy                         0.8521      4563\n",
            "   macro avg     0.6845    0.5947    0.6187      4563\n",
            "weighted avg     0.8415    0.8521    0.8444      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "sentiment_classification_eval(ytest, ypred2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rClcDOOL0LY9",
        "outputId": "3cb4afc7-2bba-4c98-d111-b4d9a7813077"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Combination Evaluate (Aspect Detection + Sentiment Classification) ##\n",
            "                                 precision    recall  f1-score   support\n",
            "\n",
            "          FACILITY#GENERAL,None     0.8459    0.8946    0.8696       313\n",
            "      FACILITY#GENERAL,negative     0.7315    0.7453    0.7383       106\n",
            "       FACILITY#GENERAL,neutral     1.0000    0.0714    0.1333        14\n",
            "      FACILITY#GENERAL,positive     0.6866    0.6216    0.6525        74\n",
            "               FOOD#OPTION,None     0.9752    0.9937    0.9844       475\n",
            "           FOOD#OPTION,negative     0.6923    0.5000    0.5806        18\n",
            "            FOOD#OPTION,neutral     1.0000    0.0000    0.0000         2\n",
            "           FOOD#OPTION,positive     0.8000    0.6667    0.7273        12\n",
            "              FOOD#QUALITY,None     0.9345    0.9741    0.9539       425\n",
            "          FOOD#QUALITY,negative     0.6250    0.5000    0.5556        20\n",
            "           FOOD#QUALITY,neutral     0.5000    0.2353    0.3200        17\n",
            "          FOOD#QUALITY,positive     0.8000    0.7111    0.7529        45\n",
            "    HOTEL_ROOM#CLEANLINESS,None     0.9293    0.9200    0.9246       300\n",
            "HOTEL_ROOM#CLEANLINESS,negative     0.6852    0.6727    0.6789        55\n",
            " HOTEL_ROOM#CLEANLINESS,neutral     1.0000    0.0000    0.0000         7\n",
            "HOTEL_ROOM#CLEANLINESS,positive     0.8654    0.9310    0.8970       145\n",
            "        HOTEL_ROOM#GENERAL,None     0.8300    0.9592    0.8899       392\n",
            "    HOTEL_ROOM#GENERAL,negative     0.3750    0.1071    0.1667        28\n",
            "     HOTEL_ROOM#GENERAL,neutral     0.6000    0.2000    0.3000        15\n",
            "    HOTEL_ROOM#GENERAL,positive     0.6341    0.3611    0.4602        72\n",
            "         HOTEL_ROOM#PRICES,None     0.9379    0.9769    0.9570       433\n",
            "     HOTEL_ROOM#PRICES,negative     0.0000    0.0000    0.0000        20\n",
            "      HOTEL_ROOM#PRICES,neutral     1.0000    0.0000    0.0000         5\n",
            "     HOTEL_ROOM#PRICES,positive     0.6296    0.6939    0.6602        49\n",
            "        HOTEL_ROOM#QUALITY,None     0.8362    0.8081    0.8219       297\n",
            "    HOTEL_ROOM#QUALITY,negative     0.5714    0.6061    0.5882        66\n",
            "     HOTEL_ROOM#QUALITY,neutral     0.1250    0.1250    0.1250         8\n",
            "    HOTEL_ROOM#QUALITY,positive     0.7535    0.7868    0.7698       136\n",
            "          LOCATION#GENERAL,None     0.8504    0.9606    0.9021       355\n",
            "      LOCATION#GENERAL,negative     0.0000    0.0000    0.0000        10\n",
            "       LOCATION#GENERAL,neutral     1.0000    0.0833    0.1538        12\n",
            "      LOCATION#GENERAL,positive     0.7692    0.6154    0.6838       130\n",
            "           SERVICE#GENERAL,None     0.8377    0.8421    0.8399       190\n",
            "       SERVICE#GENERAL,negative     0.6250    0.6707    0.6471        82\n",
            "        SERVICE#GENERAL,neutral     0.2500    0.0714    0.1111        14\n",
            "       SERVICE#GENERAL,positive     0.8661    0.8778    0.8719       221\n",
            "\n",
            "                       accuracy                         0.8521      4563\n",
            "                      macro avg     0.7101    0.5329    0.5477      4563\n",
            "                   weighted avg     0.8413    0.8521    0.8399      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "combination_eval(ytest, ypred2)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ruWhrvluNSPT"
      },
      "source": [
        "## Non-Linear SVM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jQB-5XHKwgbB",
        "outputId": "b83df14e-3402-4167-830b-9bdf85c14da6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[I 2023-06-30 03:14:45,296] A new study created in memory with name: no-name-3cbe3142-cf74-4ea1-a4fe-afae273cd107\n",
            "[I 2023-06-30 03:14:49,904] Trial 0 finished with value: 0.2453 and parameters: {'class_weight': None, 'kernel': 'poly', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:14:55,113] Trial 1 finished with value: 0.2453 and parameters: {'class_weight': None, 'kernel': 'poly', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:15:11,568] Trial 2 finished with value: 0.0284 and parameters: {'class_weight': 'balanced', 'kernel': 'rbf', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:15:16,522] Trial 3 finished with value: 0.2453 and parameters: {'class_weight': None, 'kernel': 'rbf', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:15:21,943] Trial 4 finished with value: 0.2453 and parameters: {'class_weight': None, 'kernel': 'sigmoid', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:15:38,366] Trial 5 finished with value: 0.0284 and parameters: {'class_weight': 'balanced', 'kernel': 'sigmoid', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:15:43,050] Trial 6 finished with value: 0.2453 and parameters: {'class_weight': None, 'kernel': 'poly', 'gamma': 'auto'}. Best is trial 0 with value: 0.2453.\n",
            "[I 2023-06-30 03:15:57,554] Trial 7 finished with value: 0.6667 and parameters: {'class_weight': 'balanced', 'kernel': 'rbf', 'gamma': 'scale'}. Best is trial 7 with value: 0.6667.\n",
            "[I 2023-06-30 03:16:14,942] Trial 8 finished with value: 0.0284 and parameters: {'class_weight': 'balanced', 'kernel': 'sigmoid', 'gamma': 'auto'}. Best is trial 7 with value: 0.6667.\n",
            "[I 2023-06-30 03:16:21,909] Trial 9 finished with value: 0.7007 and parameters: {'class_weight': None, 'kernel': 'sigmoid', 'gamma': 'scale'}. Best is trial 9 with value: 0.7007.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'C': 1.0, 'break_ties': False, 'cache_size': 200, 'class_weight': None, 'coef0': 0.0, 'decision_function_shape': 'ovr', 'degree': 3, 'gamma': 'scale', 'kernel': 'sigmoid', 'max_iter': 3000, 'probability': False, 'random_state': 5, 'shrinking': True, 'tol': 0.001, 'verbose': False}\n",
            "{'class_weight': None, 'kernel': 'sigmoid', 'gamma': 'scale'}\n"
          ]
        }
      ],
      "source": [
        "from sklearn.svm import SVC\n",
        "\n",
        "def svc_objective(trial):\n",
        "    params = dict(\n",
        "        class_weight=trial.suggest_categorical('class_weight', ['balanced', None]),\n",
        "        kernel=trial.suggest_categorical('kernel', ['poly', 'rbf', 'sigmoid']),\n",
        "        gamma=trial.suggest_categorical('gamma', ['auto', 'scale']),\n",
        "        max_iter=3000,\n",
        "        random_state=5\n",
        "    )\n",
        "\n",
        "    clf = MOC(SVC(**params))\n",
        "    clf.fit(xtrain_tfidf, ytrain)\n",
        "    trial.set_user_attr(key=\"model\", value=clf)\n",
        "\n",
        "    y_pred = clf.predict(xdev_tfidf)\n",
        "    return quick_f1(ydev_ml, y_pred)\n",
        "\n",
        "sampler = TPESampler(seed=22)\n",
        "svc_study = optuna.create_study(direction='maximize')\n",
        "svc_study.optimize(svc_objective, n_trials=10, callbacks=[callback])\n",
        "\n",
        "\n",
        "clf3 = svc_study.user_attrs['best_model']\n",
        "\n",
        "print(clf3.estimators_[0].get_params())\n",
        "print(svc_study.best_params)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q7X8mZcB1hyZ"
      },
      "outputs": [],
      "source": [
        "ypred3=clf3.predict(xtest_tfidf)\n",
        "ypred3= pd.DataFrame(ypred3, columns= categories)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "is6FT_Li1hyZ",
        "outputId": "9f8212c1-3f27-411d-84b1-9f11f9dcc438"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Aspect Detection Evaluate ##\n",
            "                        precision    recall  f1-score   support\n",
            "\n",
            "                           0.8608    0.9648    0.9098      3180\n",
            "      FACILITY#GENERAL     0.8478    0.6031    0.7048       194\n",
            "           FOOD#OPTION     0.8824    0.4688    0.6122        32\n",
            "          FOOD#QUALITY     0.9500    0.4634    0.6230        82\n",
            "HOTEL_ROOM#CLEANLINESS     0.9240    0.7633    0.8360       207\n",
            "    HOTEL_ROOM#GENERAL     0.8333    0.1739    0.2878       115\n",
            "     HOTEL_ROOM#PRICES     0.8298    0.5270    0.6446        74\n",
            "    HOTEL_ROOM#QUALITY     0.8187    0.6667    0.7349       210\n",
            "      LOCATION#GENERAL     0.9438    0.5526    0.6971       152\n",
            "       SERVICE#GENERAL     0.9139    0.8707    0.8918       317\n",
            "\n",
            "              accuracy                         0.8668      4563\n",
            "             macro avg     0.8805    0.6054    0.6942      4563\n",
            "          weighted avg     0.8682    0.8668    0.8542      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "aspect_detection_eval(ytest, ypred3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "arXnkfFE1hya",
        "outputId": "ce5c3786-a723-4540-ac05-0ba08c233821"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Sentiment Classification Evaluate ##\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "        None     0.8608    0.9648    0.9098      3180\n",
            "    negative     0.7530    0.4667    0.5762       405\n",
            "     neutral     0.5000    0.0106    0.0208        94\n",
            "    positive     0.8284    0.6991    0.7583       884\n",
            "\n",
            "    accuracy                         0.8494      4563\n",
            "   macro avg     0.7356    0.5353    0.5663      4563\n",
            "weighted avg     0.8375    0.8494    0.8326      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "sentiment_classification_eval(ytest, ypred3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GNbOTxEH1hya",
        "outputId": "e931be73-5ae1-40b6-a973-af802552a08d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "## Combination Evaluate (Aspect Detection + Sentiment Classification) ##\n",
            "                                 precision    recall  f1-score   support\n",
            "\n",
            "          FACILITY#GENERAL,None     0.7913    0.9329    0.8563       313\n",
            "      FACILITY#GENERAL,negative     0.7471    0.6132    0.6736       106\n",
            "       FACILITY#GENERAL,neutral     1.0000    0.0000    0.0000        14\n",
            "      FACILITY#GENERAL,positive     0.7451    0.5135    0.6080        74\n",
            "               FOOD#OPTION,None     0.9653    0.9958    0.9803       475\n",
            "           FOOD#OPTION,negative     0.7273    0.4444    0.5517        18\n",
            "            FOOD#OPTION,neutral     1.0000    0.0000    0.0000         2\n",
            "           FOOD#OPTION,positive     1.0000    0.5000    0.6667        12\n",
            "              FOOD#QUALITY,None     0.9058    0.9953    0.9484       425\n",
            "          FOOD#QUALITY,negative     0.8000    0.4000    0.5333        20\n",
            "           FOOD#QUALITY,neutral     0.5000    0.0588    0.1053        17\n",
            "          FOOD#QUALITY,positive     0.9286    0.5778    0.7123        45\n",
            "    HOTEL_ROOM#CLEANLINESS,None     0.8542    0.9567    0.9025       300\n",
            "HOTEL_ROOM#CLEANLINESS,negative     0.8000    0.4364    0.5647        55\n",
            " HOTEL_ROOM#CLEANLINESS,neutral     1.0000    0.0000    0.0000         7\n",
            "HOTEL_ROOM#CLEANLINESS,positive     0.8936    0.8690    0.8811       145\n",
            "        HOTEL_ROOM#GENERAL,None     0.8033    0.9898    0.8869       392\n",
            "    HOTEL_ROOM#GENERAL,negative     1.0000    0.0000    0.0000        28\n",
            "     HOTEL_ROOM#GENERAL,neutral     1.0000    0.0000    0.0000        15\n",
            "    HOTEL_ROOM#GENERAL,positive     0.7500    0.2500    0.3750        72\n",
            "         HOTEL_ROOM#PRICES,None     0.9239    0.9815    0.9518       433\n",
            "     HOTEL_ROOM#PRICES,negative     1.0000    0.0000    0.0000        20\n",
            "      HOTEL_ROOM#PRICES,neutral     1.0000    0.0000    0.0000         5\n",
            "     HOTEL_ROOM#PRICES,positive     0.6383    0.6122    0.6250        49\n",
            "        HOTEL_ROOM#QUALITY,None     0.7917    0.8956    0.8404       297\n",
            "    HOTEL_ROOM#QUALITY,negative     0.7333    0.5000    0.5946        66\n",
            "     HOTEL_ROOM#QUALITY,neutral     1.0000    0.0000    0.0000         8\n",
            "    HOTEL_ROOM#QUALITY,positive     0.8016    0.7426    0.7710       136\n",
            "          LOCATION#GENERAL,None     0.8373    0.9859    0.9056       355\n",
            "      LOCATION#GENERAL,negative     1.0000    0.0000    0.0000        10\n",
            "       LOCATION#GENERAL,neutral     1.0000    0.0000    0.0000        12\n",
            "      LOCATION#GENERAL,positive     0.8539    0.5846    0.6941       130\n",
            "           SERVICE#GENERAL,None     0.8000    0.8632    0.8304       190\n",
            "       SERVICE#GENERAL,negative     0.7500    0.6220    0.6800        82\n",
            "        SERVICE#GENERAL,neutral     1.0000    0.0000    0.0000        14\n",
            "       SERVICE#GENERAL,positive     0.8419    0.8914    0.8659       221\n",
            "\n",
            "                       accuracy                         0.8494      4563\n",
            "                      macro avg     0.8662    0.4781    0.5001      4563\n",
            "                   weighted avg     0.8508    0.8494    0.8268      4563\n",
            "\n"
          ]
        }
      ],
      "source": [
        "combination_eval(ytest, ypred3)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "tz4aAURfgKIW",
        "ppKYl2LZPra0",
        "6quUiO0590xf"
      ],
      "gpuType": "T4",
      "provenance": []
    },
    "deepnote": {
      "is_reactive": false
    },
    "deepnote_execution_queue": [],
    "deepnote_notebook_id": "89096161-8db2-4c83-bd09-f2de34fe0e65",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "orig_nbformat": 2
  },
  "nbformat": 4,
  "nbformat_minor": 0
}