TextAttack.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.11"
    },
    "colab": {
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "YQ6UmG-GblQJ"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-e84g1YoseoE"
      },
      "source": [
        "# TextAttack End-to-End\n",
        " end-to-end overview of training, evaluating, and attacking a model using [TextAttack](https://textattack.readthedocs.io/en/master/)."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "aZ_AIFuFSfCr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AQTkpf9RslEA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cead5c31-5e6c-481f-a812-a7d12b2a19d0"
      },
      "source": [
        "!pip install textattack[tensorflow,optional]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: textattack[optional,tensorflow] in /usr/local/lib/python3.10/dist-packages (0.3.10)\n",
            "Requirement already satisfied: bert-score>=0.3.5 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.3.13)\n",
            "Requirement already satisfied: editdistance in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.8.1)\n",
            "Requirement already satisfied: flair in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.14.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (3.16.1)\n",
            "Requirement already satisfied: language-tool-python in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.8.1)\n",
            "Requirement already satisfied: lemminflect in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.2.3)\n",
            "Requirement already satisfied: lru-dict in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (1.3.0)\n",
            "Requirement already satisfied: datasets>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (3.0.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (3.8.1)\n",
            "Requirement already satisfied: numpy>=1.21.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (1.23.5)\n",
            "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.2.2)\n",
            "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (1.13.1)\n",
            "Requirement already satisfied: torch!=1.8,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.4.1+cu121)\n",
            "Requirement already satisfied: transformers>=4.30.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (4.44.2)\n",
            "Requirement already satisfied: terminaltables in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (3.1.10)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (4.66.5)\n",
            "Requirement already satisfied: word2number in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (1.1)\n",
            "Requirement already satisfied: num2words in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.5.13)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (10.5.0)\n",
            "Requirement already satisfied: pinyin>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.4.0)\n",
            "Requirement already satisfied: jieba in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.42.1)\n",
            "Requirement already satisfied: OpenHowNet in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.0)\n",
            "Requirement already satisfied: tensorflow>=2.9.1 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.12.0)\n",
            "Requirement already satisfied: tensorflow-hub in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.16.1)\n",
            "Requirement already satisfied: tensorflow-text>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.17.0)\n",
            "Requirement already satisfied: tensorboardX in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.6.2.2)\n",
            "Requirement already satisfied: tensorflow-estimator>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.12.0)\n",
            "Requirement already satisfied: sentence-transformers==2.2.0 in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (2.2.0)\n",
            "Requirement already satisfied: stanza in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (1.9.2)\n",
            "Requirement already satisfied: visdom in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.2.4)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (0.18.3)\n",
            "Requirement already satisfied: gensim in /usr/local/lib/python3.10/dist-packages (from textattack[optional,tensorflow]) (4.3.3)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers==2.2.0->textattack[optional,tensorflow]) (0.19.1+cu121)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers==2.2.0->textattack[optional,tensorflow]) (1.5.2)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from sentence-transformers==2.2.0->textattack[optional,tensorflow]) (0.2.0)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from sentence-transformers==2.2.0->textattack[optional,tensorflow]) (0.24.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from bert-score>=0.3.5->textattack[optional,tensorflow]) (2.32.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from bert-score>=0.3.5->textattack[optional,tensorflow]) (3.7.1)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from bert-score>=0.3.5->textattack[optional,tensorflow]) (24.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.4.0->textattack[optional,tensorflow]) (16.1.0)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.4.0->textattack[optional,tensorflow]) (0.3.8)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.4.0->textattack[optional,tensorflow]) (3.5.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets>=2.4.0->textattack[optional,tensorflow]) (0.70.16)\n",
            "Requirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets>=2.4.0->textattack[optional,tensorflow]) (2024.6.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.4.0->textattack[optional,tensorflow]) (3.10.8)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.4.0->textattack[optional,tensorflow]) (6.0.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->textattack[optional,tensorflow]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->textattack[optional,tensorflow]) (2024.2)\n",
            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.0.1->textattack[optional,tensorflow]) (2024.2)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (24.3.25)\n",
            "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.4.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.2.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (1.64.1)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (3.11.0)\n",
            "Requirement already satisfied: jax>=0.3.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.4.30)\n",
            "Requirement already satisfied: keras<2.13,>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (2.12.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (18.1.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (3.4.0)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (71.0.4)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (1.16.0)\n",
            "Requirement already satisfied: tensorboard<2.13,>=2.12 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (2.12.3)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (2.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (4.12.2)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (1.14.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.37.1)\n",
            "Collecting tensorflow>=2.9.1 (from textattack[optional,tensorflow])\n",
            "  Using cached tensorflow-2.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.2 kB)\n",
            "Requirement already satisfied: ml-dtypes<0.5.0,>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.4.1)\n",
            "Collecting tensorboard<2.18,>=2.17 (from tensorflow>=2.9.1->textattack[optional,tensorflow])\n",
            "  Downloading tensorboard-2.17.1-py3-none-any.whl.metadata (1.6 kB)\n",
            "Collecting keras>=3.2.0 (from tensorflow>=2.9.1->textattack[optional,tensorflow])\n",
            "  Downloading keras-3.6.0-py3-none-any.whl.metadata (5.8 kB)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.7.0->textattack[optional,tensorflow]) (1.13.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.7.0->textattack[optional,tensorflow]) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.8,>=1.7.0->textattack[optional,tensorflow]) (3.1.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.30.0->textattack[optional,tensorflow]) (2024.9.11)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.30.0->textattack[optional,tensorflow]) (0.4.5)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.30.0->textattack[optional,tensorflow]) (0.19.1)\n",
            "Requirement already satisfied: boto3>=1.20.27 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (1.35.36)\n",
            "Requirement already satisfied: conllu<5.0.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (4.5.3)\n",
            "Requirement already satisfied: deprecated>=1.2.13 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (1.2.14)\n",
            "Requirement already satisfied: ftfy>=6.1.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (6.2.3)\n",
            "Requirement already satisfied: gdown>=4.4.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (5.2.0)\n",
            "Requirement already satisfied: langdetect>=1.0.9 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (1.0.9)\n",
            "Requirement already satisfied: lxml>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (4.9.4)\n",
            "Requirement already satisfied: mpld3>=0.3 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (0.5.10)\n",
            "Requirement already satisfied: pptree>=3.1 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (3.1)\n",
            "Requirement already satisfied: pytorch-revgrad>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (0.2.0)\n",
            "Requirement already satisfied: segtok>=1.5.11 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (1.5.11)\n",
            "Requirement already satisfied: sqlitedict>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (2.1.0)\n",
            "Requirement already satisfied: tabulate>=0.8.10 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (0.9.0)\n",
            "Requirement already satisfied: transformer-smaller-training-vocab>=0.2.3 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (0.4.0)\n",
            "Requirement already satisfied: wikipedia-api>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (0.7.1)\n",
            "Requirement already satisfied: semver<4.0.0,>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (3.0.2)\n",
            "Requirement already satisfied: bioc<3.0.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from flair->textattack[optional,tensorflow]) (2.1)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim->textattack[optional,tensorflow]) (7.0.4)\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (from language-tool-python->textattack[optional,tensorflow]) (24.1.2)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from language-tool-python->textattack[optional,tensorflow]) (0.44.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->textattack[optional,tensorflow]) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->textattack[optional,tensorflow]) (1.4.2)\n",
            "Requirement already satisfied: docopt>=0.6.2 in /usr/local/lib/python3.10/dist-packages (from num2words->textattack[optional,tensorflow]) (0.6.2)\n",
            "Requirement already satisfied: anytree in /usr/local/lib/python3.10/dist-packages (from OpenHowNet->textattack[optional,tensorflow]) (2.12.1)\n",
            "Requirement already satisfied: emoji in /usr/local/lib/python3.10/dist-packages (from stanza->textattack[optional,tensorflow]) (2.14.0)\n",
            "Requirement already satisfied: tomli in /usr/local/lib/python3.10/dist-packages (from stanza->textattack[optional,tensorflow]) (2.0.2)\n",
            "Requirement already satisfied: tf-keras>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow-hub->textattack[optional,tensorflow]) (2.17.0)\n",
            "Requirement already satisfied: tornado in /usr/local/lib/python3.10/dist-packages (from visdom->textattack[optional,tensorflow]) (6.3.3)\n",
            "Requirement already satisfied: jsonpatch in /usr/local/lib/python3.10/dist-packages (from visdom->textattack[optional,tensorflow]) (1.33)\n",
            "Requirement already satisfied: websocket-client in /usr/local/lib/python3.10/dist-packages (from visdom->textattack[optional,tensorflow]) (1.8.0)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from visdom->textattack[optional,tensorflow]) (10.4.0)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->textattack[optional,tensorflow]) (0.4.0)\n",
            "Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->textattack[optional,tensorflow]) (3.1.43)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from wandb->textattack[optional,tensorflow]) (4.3.6)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->textattack[optional,tensorflow]) (5.9.5)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->textattack[optional,tensorflow]) (2.16.0)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->textattack[optional,tensorflow]) (1.3.3)\n",
            "Requirement already satisfied: jsonlines>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from bioc<3.0.0,>=2.0.0->flair->textattack[optional,tensorflow]) (4.0.0)\n",
            "Requirement already satisfied: intervaltree in /usr/local/lib/python3.10/dist-packages (from bioc<3.0.0,>=2.0.0->flair->textattack[optional,tensorflow]) (3.1.0)\n",
            "Requirement already satisfied: botocore<1.36.0,>=1.35.36 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.20.27->flair->textattack[optional,tensorflow]) (1.35.36)\n",
            "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.20.27->flair->textattack[optional,tensorflow]) (1.0.1)\n",
            "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.20.27->flair->textattack[optional,tensorflow]) (0.10.3)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (2.4.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (6.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (1.13.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.4.0->textattack[optional,tensorflow]) (4.0.3)\n",
            "Requirement already satisfied: wcwidth<0.3.0,>=0.2.12 in /usr/local/lib/python3.10/dist-packages (from ftfy>=6.1.0->flair->textattack[optional,tensorflow]) (0.2.13)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from gdown>=4.4.0->flair->textattack[optional,tensorflow]) (4.12.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from gitpython!=3.1.29,>=1.0.0->wandb->textattack[optional,tensorflow]) (4.0.11)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow>=2.9.1->textattack[optional,tensorflow]) (13.9.1)\n",
            "Requirement already satisfied: namex in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.0.8)\n",
            "Requirement already satisfied: optree in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.13.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[optional,tensorflow]) (1.3.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[optional,tensorflow]) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[optional,tensorflow]) (4.54.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[optional,tensorflow]) (1.4.7)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->bert-score>=0.3.5->textattack[optional,tensorflow]) (3.1.4)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score>=0.3.5->textattack[optional,tensorflow]) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score>=0.3.5->textattack[optional,tensorflow]) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score>=0.3.5->textattack[optional,tensorflow]) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->bert-score>=0.3.5->textattack[optional,tensorflow]) (2024.8.30)\n",
            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers==2.2.0->textattack[optional,tensorflow]) (3.5.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow>=2.9.1->textattack[optional,tensorflow]) (3.7)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow>=2.9.1->textattack[optional,tensorflow]) (3.0.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.8,>=1.7.0->textattack[optional,tensorflow]) (2.1.5)\n",
            "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch->visdom->textattack[optional,tensorflow]) (3.0.0)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.8,>=1.7.0->textattack[optional,tensorflow]) (1.3.0)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb->textattack[optional,tensorflow]) (5.0.1)\n",
            "Requirement already satisfied: accelerate>=0.21.0 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece,torch]<5.0,>=4.1->transformer-smaller-training-vocab>=0.2.3->flair->textattack[optional,tensorflow]) (0.34.2)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->gdown>=4.4.0->flair->textattack[optional,tensorflow]) (2.6)\n",
            "Requirement already satisfied: sortedcontainers<3.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from intervaltree->bioc<3.0.0,>=2.0.0->flair->textattack[optional,tensorflow]) (2.4.0)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->gdown>=4.4.0->flair->textattack[optional,tensorflow]) (1.7.1)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow>=2.9.1->textattack[optional,tensorflow]) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow>=2.9.1->textattack[optional,tensorflow]) (2.18.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->keras>=3.2.0->tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.1.2)\n",
            "Requirement already satisfied: jaxlib<=0.4.30,>=0.4.27 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow>=2.9.1->textattack[optional,tensorflow]) (0.4.30)\n",
            "Using cached tensorflow-2.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (601.3 MB)\n",
            "Downloading keras-3.6.0-py3-none-any.whl (1.2 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m42.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading tensorboard-2.17.1-py3-none-any.whl (5.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m83.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tensorboard, keras, tensorflow\n",
            "  Attempting uninstall: tensorboard\n",
            "    Found existing installation: tensorboard 2.12.3\n",
            "    Uninstalling tensorboard-2.12.3:\n",
            "      Successfully uninstalled tensorboard-2.12.3\n",
            "  Attempting uninstall: keras\n",
            "    Found existing installation: keras 2.12.0\n",
            "    Uninstalling keras-2.12.0:\n",
            "      Successfully uninstalled keras-2.12.0\n",
            "  Attempting uninstall: tensorflow\n",
            "    Found existing installation: tensorflow 2.12.0\n",
            "    Uninstalling tensorflow-2.12.0:\n",
            "      Successfully uninstalled tensorflow-2.12.0\n",
            "Successfully installed keras-3.6.0 tensorboard-2.17.1 tensorflow-2.17.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow==2.12"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bRU0ibw3etFs",
        "outputId": "332cd270-141a-43d6-bbac-bab5f659c028"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tensorflow==2.12\n",
            "  Using cached tensorflow-2.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.4 kB)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (24.3.25)\n",
            "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (0.4.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (0.2.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (1.64.1)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (3.11.0)\n",
            "Requirement already satisfied: jax>=0.3.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (0.4.30)\n",
            "Collecting keras<2.13,>=2.12.0 (from tensorflow==2.12)\n",
            "  Using cached keras-2.12.0-py2.py3-none-any.whl.metadata (1.4 kB)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (18.1.1)\n",
            "Requirement already satisfied: numpy<1.24,>=1.22 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (1.23.5)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (3.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (24.1)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (71.0.4)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (1.16.0)\n",
            "Collecting tensorboard<2.13,>=2.12 (from tensorflow==2.12)\n",
            "  Using cached tensorboard-2.12.3-py3-none-any.whl.metadata (1.8 kB)\n",
            "Requirement already satisfied: tensorflow-estimator<2.13,>=2.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (2.12.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (2.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (4.12.2)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (1.14.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow==2.12) (0.37.1)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow==2.12) (0.44.0)\n",
            "Requirement already satisfied: jaxlib<=0.4.30,>=0.4.27 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow==2.12) (0.4.30)\n",
            "Requirement already satisfied: ml-dtypes>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow==2.12) (0.4.1)\n",
            "Requirement already satisfied: scipy>=1.9 in /usr/local/lib/python3.10/dist-packages (from jax>=0.3.15->tensorflow==2.12) (1.13.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12) (1.0.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12) (3.7)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12) (2.32.3)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.13,>=2.12->tensorflow==2.12) (3.0.4)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12) (5.5.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12) (0.4.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow==2.12) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.13,>=2.12->tensorflow==2.12) (2024.8.30)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.13,>=2.12->tensorflow==2.12) (2.1.5)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.13,>=2.12->tensorflow==2.12) (0.6.1)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.13,>=2.12->tensorflow==2.12) (3.2.2)\n",
            "Using cached tensorflow-2.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (585.9 MB)\n",
            "Using cached keras-2.12.0-py2.py3-none-any.whl (1.7 MB)\n",
            "Using cached tensorboard-2.12.3-py3-none-any.whl (5.6 MB)\n",
            "Installing collected packages: keras, tensorboard, tensorflow\n",
            "  Attempting uninstall: keras\n",
            "    Found existing installation: keras 3.6.0\n",
            "    Uninstalling keras-3.6.0:\n",
            "      Successfully uninstalled keras-3.6.0\n",
            "  Attempting uninstall: tensorboard\n",
            "    Found existing installation: tensorboard 2.17.1\n",
            "    Uninstalling tensorboard-2.17.1:\n",
            "      Successfully uninstalled tensorboard-2.17.1\n",
            "  Attempting uninstall: tensorflow\n",
            "    Found existing installation: tensorflow 2.17.0\n",
            "    Uninstalling tensorflow-2.17.0:\n",
            "      Successfully uninstalled tensorflow-2.17.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow-text 2.17.0 requires tensorflow<2.18,>=2.17.0, but you have tensorflow 2.12.0 which is incompatible.\n",
            "tf-keras 2.17.0 requires tensorflow<2.18,>=2.17, but you have tensorflow 2.12.0 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed keras-2.12.0 tensorboard-2.12.3 tensorflow-2.12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ONayD5EJseoG"
      },
      "source": [
        "## Training\n",
        "Text attack comes with its own fine tuned models on several datasets. You can list them with the command below."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! textattack list models"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "thVtyRTkGeFu",
        "outputId": "5ce5edb2-30a7-452a-dc4a-406a45b5569c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34;1mtextattack\u001b[0m: Updating TextAttack package dependencies.\n",
            "\u001b[34;1mtextattack\u001b[0m: Downloading NLTK required packages.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package omw to /root/nltk_data...\n",
            "[nltk_data] Downloading package universal_tagset to /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.9.0.json: 392kB [00:00, 154MB/s]         \n",
            "2024-10-09 12:18:25 INFO: Downloaded file to /root/stanza_resources/resources.json\n",
            "2024-10-09 12:18:25 INFO: Downloading default packages for language: en (English) ...\n",
            "Downloading https://huggingface.co/stanfordnlp/stanza-en/resolve/v1.9.0/models/default.zip: 100% 526M/526M [00:02<00:00, 214MB/s]\n",
            "2024-10-09 12:18:30 INFO: Downloaded file to /root/stanza_resources/en/default.zip\n",
            "2024-10-09 12:18:37 INFO: Finished downloading models and saved to /root/stanza_resources\n",
            "2024-10-09 12:18:40.977967: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:18:41.060205: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:18:41.060992: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-10-09 12:18:42.529461: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[94malbert-base-v2\u001b[0m\n",
            "\u001b[94malbert-base-v2-ag-news\u001b[0m\n",
            "\u001b[94malbert-base-v2-cola\u001b[0m\n",
            "\u001b[94malbert-base-v2-imdb\u001b[0m\n",
            "\u001b[94malbert-base-v2-mr\u001b[0m\n",
            "\u001b[94malbert-base-v2-qqp\u001b[0m\n",
            "\u001b[94malbert-base-v2-rte\u001b[0m\n",
            "\u001b[94malbert-base-v2-snli\u001b[0m\n",
            "\u001b[94malbert-base-v2-sst2\u001b[0m\n",
            "\u001b[94malbert-base-v2-stsb\u001b[0m\n",
            "\u001b[94malbert-base-v2-wnli\u001b[0m\n",
            "\u001b[94malbert-base-v2-yelp\u001b[0m\n",
            "\u001b[94mbert-base-uncased\u001b[0m\n",
            "\u001b[94mbert-base-uncased-ag-news\u001b[0m\n",
            "\u001b[94mbert-base-uncased-cola\u001b[0m\n",
            "\u001b[94mbert-base-uncased-imdb\u001b[0m\n",
            "\u001b[94mbert-base-uncased-mnli\u001b[0m\n",
            "\u001b[94mbert-base-uncased-mr\u001b[0m\n",
            "\u001b[94mbert-base-uncased-mrpc\u001b[0m\n",
            "\u001b[94mbert-base-uncased-qnli\u001b[0m\n",
            "\u001b[94mbert-base-uncased-qqp\u001b[0m\n",
            "\u001b[94mbert-base-uncased-rte\u001b[0m\n",
            "\u001b[94mbert-base-uncased-snli\u001b[0m\n",
            "\u001b[94mbert-base-uncased-sst2\u001b[0m\n",
            "\u001b[94mbert-base-uncased-stsb\u001b[0m\n",
            "\u001b[94mbert-base-uncased-wnli\u001b[0m\n",
            "\u001b[94mbert-base-uncased-yelp\u001b[0m\n",
            "\u001b[94mcnn-ag-news\u001b[0m\n",
            "\u001b[94mcnn-imdb\u001b[0m\n",
            "\u001b[94mcnn-mr\u001b[0m\n",
            "\u001b[94mcnn-sst2\u001b[0m\n",
            "\u001b[94mcnn-yelp\u001b[0m\n",
            "\u001b[94mdistilbert-base-cased-cola\u001b[0m\n",
            "\u001b[94mdistilbert-base-cased-mrpc\u001b[0m\n",
            "\u001b[94mdistilbert-base-cased-qqp\u001b[0m\n",
            "\u001b[94mdistilbert-base-cased-snli\u001b[0m\n",
            "\u001b[94mdistilbert-base-cased-sst2\u001b[0m\n",
            "\u001b[94mdistilbert-base-cased-stsb\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-ag-news\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-cola\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-imdb\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-mnli\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-mr\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-mrpc\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-qnli\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-rte\u001b[0m\n",
            "\u001b[94mdistilbert-base-uncased-wnli\u001b[0m\n",
            "\u001b[94mlstm-ag-news\u001b[0m\n",
            "\u001b[94mlstm-imdb\u001b[0m\n",
            "\u001b[94mlstm-mr\u001b[0m\n",
            "\u001b[94mlstm-sst2\u001b[0m\n",
            "\u001b[94mlstm-yelp\u001b[0m\n",
            "\u001b[94mroberta-base\u001b[0m\n",
            "\u001b[94mroberta-base-ag-news\u001b[0m\n",
            "\u001b[94mroberta-base-cola\u001b[0m\n",
            "\u001b[94mroberta-base-imdb\u001b[0m\n",
            "\u001b[94mroberta-base-mr\u001b[0m\n",
            "\u001b[94mroberta-base-mrpc\u001b[0m\n",
            "\u001b[94mroberta-base-qnli\u001b[0m\n",
            "\u001b[94mroberta-base-rte\u001b[0m\n",
            "\u001b[94mroberta-base-sst2\u001b[0m\n",
            "\u001b[94mroberta-base-stsb\u001b[0m\n",
            "\u001b[94mroberta-base-wnli\u001b[0m\n",
            "\u001b[94mt5-en-de\u001b[0m\n",
            "\u001b[94mt5-en-fr\u001b[0m\n",
            "\u001b[94mt5-en-ro\u001b[0m\n",
            "\u001b[94mt5-summarization\u001b[0m\n",
            "\u001b[94mxlnet-base-cased\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-cola\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-imdb\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-mr\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-mrpc\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-rte\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-stsb\u001b[0m\n",
            "\u001b[94mxlnet-base-cased-wnli\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can use these models as it is by referring to their name. However for the purpose of this practical we are going to train our model from scratch.\n",
        "\n",
        "TextAttack integrates directly with [transformers](https://github.com/huggingface/transformers/) and [datasets](https://github.com/huggingface/datasets) to train any of the `transformers` pre-trained models on datasets from `datasets`.\n",
        "\n",
        "Let's use the Rotten Tomatoes Movie Review dataset: it's relatively short, and showcases the key features of `textattack train`. Let's take a look at the dataset using `textattack peek-dataset`:"
      ],
      "metadata": {
        "id": "MhNIugCuGegY"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "spS2eW5WseoG",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "48043128-9f7d-4807-9e1b-2a7b15f33a3e"
      },
      "source": [
        "!textattack peek-dataset --dataset-from-huggingface rotten_tomatoes"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-09 12:19:03.372547: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:19:03.424446: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:19:03.425024: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-10-09 12:19:04.566450: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "README.md: 100% 7.46k/7.46k [00:00<00:00, 34.0MB/s]\n",
            "train.parquet: 100% 699k/699k [00:00<00:00, 85.7MB/s]\n",
            "validation.parquet: 100% 90.0k/90.0k [00:00<00:00, 217MB/s]\n",
            "test.parquet: 100% 92.2k/92.2k [00:00<00:00, 182MB/s]\n",
            "Generating train split: 100% 8530/8530 [00:00<00:00, 114942.71 examples/s]\n",
            "Generating validation split: 100% 1066/1066 [00:00<00:00, 336149.77 examples/s]\n",
            "Generating test split: 100% 1066/1066 [00:00<00:00, 367147.98 examples/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n",
            "\u001b[34;1mtextattack\u001b[0m: Number of samples: \u001b[94m8530\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: Number of words per input:\n",
            "\u001b[34;1mtextattack\u001b[0m: \ttotal:   \u001b[94m157755\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: \tmean:    \u001b[94m18.49\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: \tstd:     \u001b[94m8.58\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: \tmin:     \u001b[94m1\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: \tmax:     \u001b[94m51\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: Dataset lowercased: \u001b[94mTrue\u001b[0m\n",
            "\u001b[34;1mtextattack\u001b[0m: First sample:\n",
            "the rock is destined to be the 21st century's new \" conan \" and that he's going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal . \n",
            "\n",
            "\u001b[34;1mtextattack\u001b[0m: Last sample:\n",
            "things really get weird , though not particularly scary : the movie is all portent and no content . \n",
            "\n",
            "\u001b[34;1mtextattack\u001b[0m: Found 2 distinct outputs.\n",
            "\u001b[34;1mtextattack\u001b[0m: Most common outputs:\n",
            "\t 1      (4265)\n",
            "\t 0      (4265)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uguqpjnLseoI"
      },
      "source": [
        "The dataset looks good! It's lowercased already, so we'll make sure our model is uncased. The longest input is 51 words, so we can cap our maximum sequence length (`--model-max-length`) at 64.\n",
        "\n",
        "We'll train [`distilbert-base-uncased`](https://huggingface.co/transformers/model_doc/distilbert.html), since it's a relatively small model, and a good example of how we integrate with `transformers`.\n",
        "\n",
        "So we have our command:\n",
        "\n",
        "```bash\n",
        "textattack train                      \\ # Train a model with TextAttack\n",
        "    --model distilbert-base-uncased   \\ # Using distilbert, uncased version, from `transformers`\n",
        "    --dataset rotten_tomatoes         \\ # On the Rotten Tomatoes dataset\n",
        "    --model-num-labels 2              \\ # That has 2 labels\n",
        "    --model-max-length 64             \\ # With a maximum sequence length of 64\n",
        "    --per-device-train-batch-size 128 \\ # And batch size of 128\n",
        "    --num-epochs 3                    \\ # For 3 epochs\n",
        "```\n",
        "\n",
        "Now let's run it (please remember to use GPU if you have access):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BY33W9aWseoI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bcb74ee2-8455-41cc-9858-70da1e5d2853"
      },
      "source": [
        "!textattack train --model-name-or-path distilbert-base-uncased --dataset rotten_tomatoes --model-num-labels 2 --model-max-length 64 --per-device-train-batch-size 128 --num-epochs 3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-09 12:19:41.009290: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:19:41.058650: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:19:41.059203: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-10-09 12:19:42.175150: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading transformers AutoModelForSequenceClassification: distilbert-base-uncased\n",
            "config.json: 100% 483/483 [00:00<00:00, 3.01MB/s]\n",
            "model.safetensors: 100% 268M/268M [00:00<00:00, 305MB/s]\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "tokenizer_config.json: 100% 48.0/48.0 [00:00<00:00, 270kB/s]\n",
            "vocab.txt: 100% 232k/232k [00:00<00:00, 9.59MB/s]\n",
            "tokenizer.json: 100% 466k/466k [00:00<00:00, 1.09MB/s]\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mvalidation\u001b[0m.\n",
            "\u001b[34;1mtextattack\u001b[0m: Writing logs to ./outputs/2024-10-09-12-19-45-963793/train_log.txt.\n",
            "\u001b[34;1mtextattack\u001b[0m: Wrote original training args to ./outputs/2024-10-09-12-19-45-963793/training_args.json.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "\u001b[34;1mtextattack\u001b[0m: ***** Running training *****\n",
            "\u001b[34;1mtextattack\u001b[0m:   Num examples = 8530\n",
            "\u001b[34;1mtextattack\u001b[0m:   Num epochs = 3\n",
            "\u001b[34;1mtextattack\u001b[0m:   Num clean epochs = 3\n",
            "\u001b[34;1mtextattack\u001b[0m:   Instantaneous batch size per device = 128\n",
            "\u001b[34;1mtextattack\u001b[0m:   Total train batch size (w. parallel, distributed & accumulation) = 128\n",
            "\u001b[34;1mtextattack\u001b[0m:   Gradient accumulation steps = 1\n",
            "\u001b[34;1mtextattack\u001b[0m:   Total optimization steps = 201\n",
            "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n",
            "\u001b[34;1mtextattack\u001b[0m: Epoch 1\n",
            "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 1/3\n",
            "Loss 0.68224: 100% 67/67 [00:39<00:00,  1.70it/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 57.15%\n",
            "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 71.86%\n",
            "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2024-10-09-12-19-45-963793/best_model/\n",
            "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n",
            "\u001b[34;1mtextattack\u001b[0m: Epoch 2\n",
            "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 2/3\n",
            "Loss 0.56928: 100% 67/67 [00:40<00:00,  1.67it/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 80.06%\n",
            "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 84.52%\n",
            "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2024-10-09-12-19-45-963793/best_model/\n",
            "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n",
            "\u001b[34;1mtextattack\u001b[0m: Epoch 3\n",
            "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 3/3\n",
            "Loss 0.48906: 100% 67/67 [00:40<00:00,  1.67it/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 86.10%\n",
            "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 85.55%\n",
            "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/2024-10-09-12-19-45-963793/best_model/\n",
            "\u001b[34;1mtextattack\u001b[0m: Wrote README to ./outputs/2024-10-09-12-19-45-963793/README.md.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4xzv3BGLseoI"
      },
      "source": [
        "## Evaluation\n",
        "\n",
        "We successfully fine-tuned `distilbert-base-cased` for 3 epochs. Now let's evaluate it using `textattack eval`. This is as simple as providing the path to the pretrained model (that you just obtain from running the above command!) to `--model`, along with the number of evaluation samples. `textattack eval` will automatically load the evaluation data from training:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qGYR_W6DseoJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "610f0756-1dd6-475c-fd4c-5b94b9655838"
      },
      "source": [
        "!textattack eval --num-examples 1000 --model ./outputs/2024-09-30-08-37-16-508338/best_model/ --dataset-from-huggingface rotten_tomatoes --dataset-split test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-09 12:23:03.491584: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:23:03.543114: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:23:03.543650: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-10-09 12:23:04.426420: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/textattack\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/commands/textattack_cli.py\", line 49, in main\n",
            "    func.run(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/commands/eval_model_command.py\", line 103, in run\n",
            "    self.test_model_on_dataset(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/commands/eval_model_command.py\", line 47, in test_model_on_dataset\n",
            "    model = ModelArgs._create_model_from_args(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/model_args.py\", line 305, in _create_model_from_args\n",
            "    raise ValueError(f\"Error: unsupported TextAttack model {args.model}\")\n",
            "ValueError: Error: unsupported TextAttack model ./outputs/2024-09-30-08-37-16-508338/best_model/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sFPkCZShseoJ"
      },
      "source": [
        "Awesome -- we were able to train a model up to 84.4% accuracy on the test dataset – with only a single command!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wWglEuvUseoK"
      },
      "source": [
        "## Attack\n",
        "\n",
        "Finally, let's attack our pre-trained model. We can do this the same way as before (by providing the path to the pretrained model to `--model`). For our attack, let's use the \"TextFooler\" attack recipe, from the paper [\"Is BERT Really Robust? A Strong Baseline for Natural Language Attack on Text Classification and Entailment\" (Jin et al, 2019)](https://arxiv.org/abs/1907.11932). We can do this by passing `--recipe textfooler` to `textattack attack`.\n",
        "\n",
        "> *Warning*: We're printing out 100 examples and, if the attack succeeds, their perturbations. The output of this command is going to be quite long!\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vL-Bo1bgseoK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c38eb16a-41c3-4f57-96c4-1f33b4c2f63c"
      },
      "source": [
        "!textattack attack --recipe textfooler --num-examples 100 --model ./outputs/2024-09-30-08-37-16-508338/best_model/ --dataset-from-huggingface rotten_tomatoes --dataset-split test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-09 12:24:06.553104: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:24:06.636873: I tensorflow/tsl/cuda/cudart_stub.cc:28] Could not find cuda drivers on your machine, GPU will not be used.\n",
            "2024-10-09 12:24:06.637542: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-10-09 12:24:08.031659: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtest\u001b[0m.\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/textattack\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/commands/textattack_cli.py\", line 49, in main\n",
            "    func.run(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/commands/attack_command.py\", line 31, in run\n",
            "    model_wrapper = ModelArgs._create_model_from_args(attack_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/textattack/model_args.py\", line 305, in _create_model_from_args\n",
            "    raise ValueError(f\"Error: unsupported TextAttack model {args.model}\")\n",
            "ValueError: Error: unsupported TextAttack model ./outputs/2024-09-30-08-37-16-508338/best_model/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iyrJM3CaseoL"
      },
      "source": [
        "Looks like our model was 84% successful (makes sense - same evaluation set as `textattack eval`!), meaning that TextAttack attacked the model with 84 examples (since the attack won't run if an example is originally mispredicted). The attack success rate was 98.8%, meaning that TextFooler failed to find an adversarial example only 1.2% (1 out of 84) of the time.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#**TO DO: Robust models**\n",
        "\n",
        "Now that we have trained our model and saw that it was vulnerable to adversarial attacks, your next task is to improve its robustness. We can do so by training a model with adversarial data instead of the normal ones.\n",
        "\n",
        "1. To do so we need to update the training command from above to instruct textattack to use adversarial data generated with textfooler or any other attack during training.\n",
        "\n",
        "To complete the task you can take the help of the documentation of [TextAttack library ](https://textattack.readthedocs.io/en/latest/0_get_started/basic-Intro.html) and command line help option to adversarially train a model on the same dataset.\n",
        "\n",
        "***Hint***: Take a look at the [Trainer class in API](https://textattack.readthedocs.io/en/master/api/trainer.html) user guide and the [Making Vanilla Adversarial Training of NLP Models Feasible!](https://textattack.readthedocs.io/en/master/1start/A2TforVanillaAT.html)\n",
        "\n",
        "2. If the solution you found is expected to take more than 15 minutes to train, look again at the docummentation and adapt your parameters such that it will take between 5-10 minutes due to time restrictions for this class.\n",
        "\n",
        "3. Evaluate if the robustnes of the model has improved by attacking the newly trained model with TextFooler"
      ],
      "metadata": {
        "id": "-xwZqGWz8UVJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!textattack train --help"
      ],
      "metadata": {
        "id": "T1FpXmrp5V6g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "741c115d-069e-4051-cd43-be695103c6d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "usage: [python -m] texattack <command> [<args>] train [-h]\n",
            "                                                      --model-name-or-path\n",
            "                                                      MODEL_NAME_OR_PATH\n",
            "                                                      [--model-max-length MODEL_MAX_LENGTH]\n",
            "                                                      [--model-num-labels MODEL_NUM_LABELS]\n",
            "                                                      [--attack ATTACK]\n",
            "                                                      [--task-type TASK_TYPE]\n",
            "                                                      --dataset DATASET\n",
            "                                                      [--dataset-train-split DATASET_TRAIN_SPLIT]\n",
            "                                                      [--dataset-eval-split DATASET_EVAL_SPLIT]\n",
            "                                                      [--filter-train-by-labels FILTER_TRAIN_BY_LABELS [FILTER_TRAIN_BY_LABELS ...]]\n",
            "                                                      [--filter-eval-by-labels FILTER_EVAL_BY_LABELS [FILTER_EVAL_BY_LABELS ...]]\n",
            "                                                      [--num-epochs NUM_EPOCHS]\n",
            "                                                      [--num-clean-epochs NUM_CLEAN_EPOCHS]\n",
            "                                                      [--attack-epoch-interval ATTACK_EPOCH_INTERVAL]\n",
            "                                                      [--early-stopping-epochs EARLY_STOPPING_EPOCHS]\n",
            "                                                      [--learning-rate LEARNING_RATE]\n",
            "                                                      [--num-warmup-steps NUM_WARMUP_STEPS]\n",
            "                                                      [--weight-decay WEIGHT_DECAY]\n",
            "                                                      [--per-device-train-batch-size PER_DEVICE_TRAIN_BATCH_SIZE]\n",
            "                                                      [--per-device-eval-batch-size PER_DEVICE_EVAL_BATCH_SIZE]\n",
            "                                                      [--gradient-accumulation-steps GRADIENT_ACCUMULATION_STEPS]\n",
            "                                                      [--random-seed RANDOM_SEED]\n",
            "                                                      [--parallel]\n",
            "                                                      [--load-best-model-at-end]\n",
            "                                                      [--alpha ALPHA]\n",
            "                                                      [--num-train-adv-examples NUM_TRAIN_ADV_EXAMPLES]\n",
            "                                                      [--query-budget-train QUERY_BUDGET_TRAIN]\n",
            "                                                      [--attack-num-workers-per-device ATTACK_NUM_WORKERS_PER_DEVICE]\n",
            "                                                      [--output-dir OUTPUT_DIR]\n",
            "                                                      [--checkpoint-interval-steps CHECKPOINT_INTERVAL_STEPS]\n",
            "                                                      [--checkpoint-interval-epochs CHECKPOINT_INTERVAL_EPOCHS]\n",
            "                                                      [--save-last]\n",
            "                                                      [--log-to-tb]\n",
            "                                                      [--tb-log-dir TB_LOG_DIR]\n",
            "                                                      [--log-to-wandb]\n",
            "                                                      [--wandb-project WANDB_PROJECT]\n",
            "                                                      [--logging-interval-step LOGGING_INTERVAL_STEP]\n",
            "\n",
            "optional arguments:\n",
            "  -h, --help            show this help message and exit\n",
            "  --model-name-or-path MODEL_NAME_OR_PATH, --model MODEL_NAME_OR_PATH\n",
            "                        Name or path of the model we want to create. \"lstm\"\n",
            "                        and \"cnn\" will create TextAttack's LSTM and CNN models\n",
            "                        while any other input will be used to create\n",
            "                        Transformers model. (e.g.\"brt-base-uncased\").\n",
            "                        (default: None)\n",
            "  --model-max-length MODEL_MAX_LENGTH\n",
            "                        The maximum sequence length of the model. (default:\n",
            "                        None)\n",
            "  --model-num-labels MODEL_NUM_LABELS\n",
            "                        The number of labels for classification. (default:\n",
            "                        None)\n",
            "  --attack ATTACK       Attack recipe to use (enables adversarial training)\n",
            "                        (default: None)\n",
            "  --task-type TASK_TYPE\n",
            "                        Type of task model is supposed to perform. Options:\n",
            "                        `classification`, `regression`. (default:\n",
            "                        classification)\n",
            "  --dataset DATASET     dataset for training; will be loaded from `datasets`\n",
            "                        library. if dataset has a subset, separate with a\n",
            "                        colon. ex: `glue^sst2` or `rotten_tomatoes` (default:\n",
            "                        yelp)\n",
            "  --dataset-train-split DATASET_TRAIN_SPLIT\n",
            "                        train dataset split, if non-standard (can\n",
            "                        automatically detect 'train' (default: )\n",
            "  --dataset-eval-split DATASET_EVAL_SPLIT\n",
            "                        val dataset split, if non-standard (can automatically\n",
            "                        detect 'dev', 'validation', 'eval') (default: )\n",
            "  --filter-train-by-labels FILTER_TRAIN_BY_LABELS [FILTER_TRAIN_BY_LABELS ...]\n",
            "                        List of labels to keep in the train dataset and\n",
            "                        discard all others. (default: None)\n",
            "  --filter-eval-by-labels FILTER_EVAL_BY_LABELS [FILTER_EVAL_BY_LABELS ...]\n",
            "                        List of labels to keep in the eval dataset and discard\n",
            "                        all others. (default: None)\n",
            "  --num-epochs NUM_EPOCHS, --epochs NUM_EPOCHS\n",
            "                        Total number of epochs for training. (default: 3)\n",
            "  --num-clean-epochs NUM_CLEAN_EPOCHS\n",
            "                        Number of epochs to train on the clean dataset before\n",
            "                        adversarial training (N/A if --attack unspecified)\n",
            "                        (default: 1)\n",
            "  --attack-epoch-interval ATTACK_EPOCH_INTERVAL\n",
            "                        Generate a new adversarial training set every N\n",
            "                        epochs. (default: 1)\n",
            "  --early-stopping-epochs EARLY_STOPPING_EPOCHS\n",
            "                        Number of epochs validation must increase before\n",
            "                        stopping early (-1 for no early stopping) (default:\n",
            "                        None)\n",
            "  --learning-rate LEARNING_RATE, --lr LEARNING_RATE\n",
            "                        Learning rate for Adam Optimization. (default: 5e-05)\n",
            "  --num-warmup-steps NUM_WARMUP_STEPS\n",
            "                        The number of steps for the warmup phase of linear\n",
            "                        scheduler. (default: 500)\n",
            "  --weight-decay WEIGHT_DECAY\n",
            "                        Weight decay (L2 penalty). (default: 0.01)\n",
            "  --per-device-train-batch-size PER_DEVICE_TRAIN_BATCH_SIZE\n",
            "                        The batch size per GPU/CPU for training. (default: 8)\n",
            "  --per-device-eval-batch-size PER_DEVICE_EVAL_BATCH_SIZE\n",
            "                        The batch size per GPU/CPU for evaluation. (default:\n",
            "                        32)\n",
            "  --gradient-accumulation-steps GRADIENT_ACCUMULATION_STEPS\n",
            "                        Number of updates steps to accumulate the gradients\n",
            "                        for, before performing a backward/update pass.\n",
            "                        (default: 1)\n",
            "  --random-seed RANDOM_SEED\n",
            "                        Random seed. (default: 786)\n",
            "  --parallel            If set, run training on multiple GPUs. (default:\n",
            "                        False)\n",
            "  --load-best-model-at-end\n",
            "                        If set, keep track of the best model across training\n",
            "                        and load it at the end. (default: False)\n",
            "  --alpha ALPHA         The weight of adversarial loss. (default: 1.0)\n",
            "  --num-train-adv-examples NUM_TRAIN_ADV_EXAMPLES\n",
            "                        The number of samples to attack when generating\n",
            "                        adversarial training set. Default is -1 (which is all\n",
            "                        possible samples). (default: -1)\n",
            "  --query-budget-train QUERY_BUDGET_TRAIN\n",
            "                        The max query budget to use when generating\n",
            "                        adversarial training set. (default: None)\n",
            "  --attack-num-workers-per-device ATTACK_NUM_WORKERS_PER_DEVICE\n",
            "                        Number of worker processes to run per device for\n",
            "                        attack. Same as `num_workers_per_device` argument for\n",
            "                        `AttackArgs`. (default: 1)\n",
            "  --output-dir OUTPUT_DIR\n",
            "                        Directory to output training logs and checkpoints.\n",
            "                        (default: ./outputs/2022-11-24-15-44-45-406385)\n",
            "  --checkpoint-interval-steps CHECKPOINT_INTERVAL_STEPS\n",
            "                        Save model checkpoint after every N updates to the\n",
            "                        model. (default: None)\n",
            "  --checkpoint-interval-epochs CHECKPOINT_INTERVAL_EPOCHS\n",
            "                        Save model checkpoint after every N epochs. (default:\n",
            "                        None)\n",
            "  --save-last           If set, save the model at end of training. Can be used\n",
            "                        with `--load-best-model-at-end` to save the best model\n",
            "                        at the end. (default: True)\n",
            "  --log-to-tb           If set, log to Tensorboard (default: False)\n",
            "  --tb-log-dir TB_LOG_DIR\n",
            "                        Path of Tensorboard log directory. (default: None)\n",
            "  --log-to-wandb        If set, log to Wandb. (default: False)\n",
            "  --wandb-project WANDB_PROJECT\n",
            "                        Name of Wandb project for logging. (default:\n",
            "                        textattack)\n",
            "  --logging-interval-step LOGGING_INTERVAL_STEP\n",
            "                        Log to Tensorboard/Wandb every N steps. (default: 1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!textattack train \\\n",
        "    --model-name-or-path distilbert-base-uncased \\\n",
        "    --model-max-length 64 \\\n",
        "    --model-num-labels 2 \\\n",
        "    --attack textfooler \\\n",
        "    --task-type classification \\\n",
        "    --dataset rotten_tomatoes \\\n",
        "    --num-epochs 3 \\\n",
        "    --num-clean-epochs 1 \\\n",
        "    --learning-rate 5e-05 \\\n",
        "    --per-device-train-batch-size 8 \\\n",
        "    --per-device-eval-batch-size 32 \\\n",
        "    --gradient-accumulation-steps 1 \\\n",
        "    --random-seed 786 \\\n",
        "    --output-dir ./outputs/distilbert_rotten_tomatoes/ \\\n",
        "    --num-train-adv-examples 1000 \\\n",
        "    --save-last \\\n",
        "    --log-to-tb \\\n",
        "    --tb-log-dir ./logs/ \\\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G5YNEYOxQWfT",
        "outputId": "4a8916b5-5f1f-41cf-a686-31076e204942"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-10-09 12:58:28.459086: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading transformers AutoModelForSequenceClassification: distilbert-base-uncased\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mtrain\u001b[0m.\n",
            "\u001b[34;1mtextattack\u001b[0m: Loading \u001b[94mdatasets\u001b[0m dataset \u001b[94mrotten_tomatoes\u001b[0m, split \u001b[94mvalidation\u001b[0m.\n",
            "\u001b[34;1mtextattack\u001b[0m: Unknown if model of class <class 'transformers.models.distilbert.modeling_distilbert.DistilBertForSequenceClassification'> compatible with goal function <class 'textattack.goal_functions.classification.untargeted_classification.UntargetedClassification'>.\n",
            "\u001b[34;1mtextattack\u001b[0m: Writing logs to ./outputs/distilbert_rotten_tomatoes/train_log.txt.\n",
            "\u001b[34;1mtextattack\u001b[0m: Wrote original training args to ./outputs/distilbert_rotten_tomatoes/training_args.json.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "\u001b[34;1mtextattack\u001b[0m: ***** Running training *****\n",
            "\u001b[34;1mtextattack\u001b[0m:   Num examples = 8530\n",
            "\u001b[34;1mtextattack\u001b[0m:   Num epochs = 3\n",
            "\u001b[34;1mtextattack\u001b[0m:   Num clean epochs = 1\n",
            "\u001b[34;1mtextattack\u001b[0m:   Instantaneous batch size per device = 8\n",
            "\u001b[34;1mtextattack\u001b[0m:   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "\u001b[34;1mtextattack\u001b[0m:   Gradient accumulation steps = 1\n",
            "\u001b[34;1mtextattack\u001b[0m:   Total optimization steps = 3451\n",
            "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n",
            "\u001b[34;1mtextattack\u001b[0m: Epoch 1\n",
            "\u001b[34;1mtextattack\u001b[0m: Running clean epoch 1/1\n",
            "Loss 0.45431: 100% 1067/1067 [01:02<00:00, 16.99it/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 78.03%\n",
            "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 82.18%\n",
            "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/distilbert_rotten_tomatoes//best_model/\n",
            "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n",
            "\u001b[34;1mtextattack\u001b[0m: Epoch 2\n",
            "\u001b[34;1mtextattack\u001b[0m: Attacking model to generate new adversarial training set...\n",
            "  0% 0/1000 [00:00<?, ?it/s]2024-10-09 13:01:00.612917: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
            "Skipping registering GPU devices...\n",
            "2024-10-09 13:01:01.666638: W tensorflow/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 34133760 exceeds 10% of free system memory.\n",
            "2024-10-09 13:01:01.694630: W tensorflow/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 34133760 exceeds 10% of free system memory.\n",
            "2024-10-09 13:01:01.723019: W tensorflow/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 34133760 exceeds 10% of free system memory.\n",
            "2024-10-09 13:01:01.751797: W tensorflow/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 34133760 exceeds 10% of free system memory.\n",
            "2024-10-09 13:01:01.780661: W tensorflow/tsl/framework/cpu_allocator_impl.cc:83] Allocation of 34133760 exceeds 10% of free system memory.\n",
            "[Succeeded / Failed / Skipped / Total] 1000 / 35 / 91 / 1126: 100% 1000/1000 [05:42<00:00,  2.92it/s]\n",
            "\n",
            "\n",
            "\u001b[34;1mtextattack\u001b[0m: Total number of attack results: 1126\n",
            "\u001b[34;1mtextattack\u001b[0m: Attack success rate: 96.62% [1000 / 1035]\n",
            "Loss 0.36143: 100% 1192/1192 [01:10<00:00, 16.80it/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 88.63%\n",
            "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 84.52%\n",
            "\u001b[34;1mtextattack\u001b[0m: Best score found. Saved model to ./outputs/distilbert_rotten_tomatoes//best_model/\n",
            "\u001b[34;1mtextattack\u001b[0m: ==========================================================\n",
            "\u001b[34;1mtextattack\u001b[0m: Epoch 3\n",
            "\u001b[34;1mtextattack\u001b[0m: Attacking model to generate new adversarial training set...\n",
            "[Succeeded / Failed / Skipped / Total] 1000 / 113 / 16 / 1129: 100% 1000/1000 [07:20<00:00,  2.27it/s]\n",
            "\n",
            "\n",
            "\u001b[34;1mtextattack\u001b[0m: Total number of attack results: 1129\n",
            "\u001b[34;1mtextattack\u001b[0m: Attack success rate: 89.85% [1000 / 1113]\n",
            "Loss 0.27552: 100% 1192/1192 [01:11<00:00, 16.77it/s]\n",
            "\u001b[34;1mtextattack\u001b[0m: Train accuracy: 95.40%\n",
            "\u001b[34;1mtextattack\u001b[0m: Eval accuracy: 83.58%\n",
            "\u001b[34;1mtextattack\u001b[0m: Wrote README to ./outputs/distilbert_rotten_tomatoes/README.md.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "Z6IwqIM4bi6h"
      }
    }
  ]
}