+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","gpuClass":"standard"},"cells":[{"cell_type":"markdown","source":["## Setup datasets on Colab - only once"],"metadata":{"id":"uJWTe9GjRHQJ"}},{"cell_type":"markdown","source":["The following code must be executed only the first time that you open Colab; it is needed to unzip the datasets that we provided into your GDrive account so that you can access them in the future\n","\n","NB: BEFORE running this notebook, you must copy the datasets zip into your GDrive. You can use the link that we provided and simply click 'create a copy'. Make sure that you have enough space (roughly 8 GBs)"],"metadata":{"id":"NhknBP6cRLTm"}},{"cell_type":"code","source":["# from google.colab import drive\n","# drive.mount('/content/drive')"],"metadata":{"id":"TwmjD6mRcPtc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# move into your mounted GDrive filesystem\n","# %cd /content/drive/MyDrive/datasets\n","\n","# unzip the datasets to /content, so you don't risk going over the GDrive storage limit\n","# this can take a few minutes\n","# !unzip -q ../drive/MyDrive/datasets/gsv_xs.zip -d ./data\n","# !unzip -q ../drive/MyDrive/datasets/tokyo_xs.zip -d ./data\n","# !unzip -q ../drive/MyDrive/datasets/sf_xs.zip -d ./data"],"metadata":{"id":"6Hc56Wxkpcp7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# now remove the zips from your GDrive\n","# %rm gsv_xs.zip tokyo_xs.zip sf_xs.zip\n","# and move the unzipped datasets into your GDrive; in this way the next time\n","# you open Colab you can directly access them\n","# !mv /content/*_xs /content/drive/MyDrive/datasets"],"metadata":{"id":"PCpsbwtuQmtT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# now download the code. It is better if you fork this repo, so that you can push your\n","# modifications. Inside the repo folder you will also find logs of the experiments\n","%cd /content\n","!git clone https://github.com/domenicoMuscill0/Simple_VPR_codebase.git\n","%cd /content/Simple_VPR_codebase"],"metadata":{"id":"1Eb674sQRo_E","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1680886404656,"user_tz":-120,"elapsed":8,"user":{"displayName":"domenico muscillo","userId":"05625114363572629197"}},"outputId":"dcfca92b-1840-4aa0-8736-11078289f8d6"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content\n","Cloning into 'Simple_VPR_codebase'...\n","remote: Enumerating objects: 60, done.\u001b[K\n","remote: Counting objects: 100% (60/60), done.\u001b[K\n","remote: Compressing objects: 100% (53/53), done.\u001b[K\n","remote: Total 60 (delta 25), reused 32 (delta 4), pack-reused 0\u001b[K\n","Unpacking objects: 100% (60/60), 19.64 KiB | 838.00 KiB/s, done.\n","/content/Simple_VPR_codebase\n"]}]},{"cell_type":"markdown","source":["## Setup environment - every time"],"metadata":{"id":"718c2aALRZzH"}},{"cell_type":"markdown","source":["Every time you restart Colab's kernel, you have to re-install packages and download anything that was not saved into your GDrive"],"metadata":{"id":"YpOkk8I0RiPU"}},{"cell_type":"code","source":["from google.colab import drive\n","drive.mount('/content/drive')"],"metadata":{"id":"u9eZYfxzSE8d","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1680886421469,"user_tz":-120,"elapsed":16816,"user":{"displayName":"domenico muscillo","userId":"05625114363572629197"}},"outputId":"efddf912-7a3f-427e-aa35-58f6e5018aaa"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}]},{"cell_type":"code","source":["%cd /content/Simple_VPR_codebase"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xJJYyGeemmGm","outputId":"ec48bf9b-7dec-4a13-cb15-e9d96d7a3cd6","executionInfo":{"status":"ok","timestamp":1680886421469,"user_tz":-120,"elapsed":8,"user":{"displayName":"domenico muscillo","userId":"05625114363572629197"}}},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/content/Simple_VPR_codebase\n"]}]},{"cell_type":"code","source":["!pip install -r requirements.txt"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"IwpebDBcDLrO","outputId":"930f6e3f-9f7e-45e9-c4d0-417e83fcda62","executionInfo":{"status":"ok","timestamp":1680886538785,"user_tz":-120,"elapsed":117321,"user":{"displayName":"domenico muscillo","userId":"05625114363572629197"}}},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/, https://download.pytorch.org/whl/cu116\n","Collecting torch==1.13.1+cu116\n"," Downloading https://download.pytorch.org/whl/cu116/torch-1.13.1%2Bcu116-cp39-cp39-linux_x86_64.whl (1977.9 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 GB\u001b[0m \u001b[31m877.6 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting torchvision==0.14.1+cu116\n"," Downloading https://download.pytorch.org/whl/cu116/torchvision-0.14.1%2Bcu116-cp39-cp39-linux_x86_64.whl (24.2 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m24.2/24.2 MB\u001b[0m \u001b[31m46.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting faiss-cpu==1.7.3\n"," Downloading faiss_cpu-1.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.0 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m78.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting pytorch-lightning==1.9.4\n"," Downloading pytorch_lightning-1.9.4-py3-none-any.whl (827 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m827.8/827.8 KB\u001b[0m \u001b[31m63.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting pytorch-metric-learning==2.0.1\n"," Downloading pytorch_metric_learning-2.0.1-py3-none-any.whl (109 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.3/109.3 KB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: opencv-python==4.7.0.72 in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 8)) (4.7.0.72)\n","Requirement already satisfied: scikit-image==0.19.3 in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 9)) (0.19.3)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch==1.13.1+cu116->-r requirements.txt (line 3)) (4.5.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (1.22.4)\n","Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.9/dist-packages (from torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (8.4.0)\n","Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (2.27.1)\n","Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning==1.9.4->-r requirements.txt (line 6)) (4.65.0)\n","Collecting lightning-utilities>=0.6.0.post0\n"," Downloading lightning_utilities-0.8.0-py3-none-any.whl (20 kB)\n","Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning==1.9.4->-r requirements.txt (line 6)) (6.0)\n","Requirement already satisfied: fsspec[http]>2021.06.0 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning==1.9.4->-r requirements.txt (line 6)) (2023.3.0)\n","Requirement already satisfied: packaging>=17.1 in /usr/local/lib/python3.9/dist-packages (from pytorch-lightning==1.9.4->-r requirements.txt (line 6)) (23.0)\n","Collecting torchmetrics>=0.7.0\n"," Downloading torchmetrics-0.11.4-py3-none-any.whl (519 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.2/519.2 KB\u001b[0m \u001b[31m47.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.9/dist-packages (from pytorch-metric-learning==2.0.1->-r requirements.txt (line 7)) (1.2.2)\n","Requirement already satisfied: PyWavelets>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from scikit-image==0.19.3->-r requirements.txt (line 9)) (1.4.1)\n","Requirement already satisfied: tifffile>=2019.7.26 in /usr/local/lib/python3.9/dist-packages (from scikit-image==0.19.3->-r requirements.txt (line 9)) (2023.3.21)\n","Requirement already satisfied: imageio>=2.4.1 in /usr/local/lib/python3.9/dist-packages (from scikit-image==0.19.3->-r requirements.txt (line 9)) (2.25.1)\n","Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.9/dist-packages (from scikit-image==0.19.3->-r requirements.txt (line 9)) (1.10.1)\n","Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.9/dist-packages (from scikit-image==0.19.3->-r requirements.txt (line 9)) (3.0)\n","Collecting aiohttp!=4.0.0a0,!=4.0.0a1\n"," Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m71.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (3.4)\n","Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (2.0.12)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (2022.12.7)\n","Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->torchvision==0.14.1+cu116->-r requirements.txt (line 4)) (1.26.15)\n","Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from scikit-learn->pytorch-metric-learning==2.0.1->-r requirements.txt (line 7)) (1.1.1)\n","Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from scikit-learn->pytorch-metric-learning==2.0.1->-r requirements.txt (line 7)) (3.1.0)\n","Collecting multidict<7.0,>=4.5\n"," Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting aiosignal>=1.1.2\n"," Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n","Collecting frozenlist>=1.1.1\n"," Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting yarl<2.0,>=1.0\n"," Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m31.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning==1.9.4->-r requirements.txt (line 6)) (22.2.0)\n","Collecting async-timeout<5.0,>=4.0.0a3\n"," Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n","Installing collected packages: faiss-cpu, torch, multidict, lightning-utilities, frozenlist, async-timeout, yarl, torchvision, torchmetrics, aiosignal, pytorch-metric-learning, aiohttp, pytorch-lightning\n"," Attempting uninstall: torch\n"," Found existing installation: torch 2.0.0+cu118\n"," Uninstalling torch-2.0.0+cu118:\n"," Successfully uninstalled torch-2.0.0+cu118\n"," Attempting uninstall: torchvision\n"," Found existing installation: torchvision 0.15.1+cu118\n"," Uninstalling torchvision-0.15.1+cu118:\n"," Successfully uninstalled torchvision-0.15.1+cu118\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","torchtext 0.15.1 requires torch==2.0.0, but you have torch 1.13.1+cu116 which is incompatible.\n","torchdata 0.6.0 requires torch==2.0.0, but you have torch 1.13.1+cu116 which is incompatible.\n","torchaudio 2.0.1+cu118 requires torch==2.0.0, but you have torch 1.13.1+cu116 which is incompatible.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 faiss-cpu-1.7.3 frozenlist-1.3.3 lightning-utilities-0.8.0 multidict-6.0.4 pytorch-lightning-1.9.4 pytorch-metric-learning-2.0.1 torch-1.13.1+cu116 torchmetrics-0.11.4 torchvision-0.14.1+cu116 yarl-1.8.2\n"]}]},{"cell_type":"code","source":["!unzip -q ../drive/MyDrive/datasets/gsv_xs.zip -d ./data\n","!unzip -q ../drive/MyDrive/datasets/tokyo_xs.zip -d ./data\n","!unzip -q ../drive/MyDrive/datasets/sf_xs.zip -d ./data\n","# !python download_datasets.py"],"metadata":{"id":"i8ztokKOWuxt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# !rm -rf ../drive/MyDrive/LOGS/lightning_logs/version_1"],"metadata":{"id":"HUx2pReASHxP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# this is an example of a basic experiments. You can choose to validate on tokyo_xs or sf_xs\n","\n","!python main.py --train_path ./data/gsv_xs --val_path ./data/tokyo_xs/test --test_path ./data/tokyo_xs/test --log_path ../drive/MyDrive/LOGS --num_preds_to_save 3 --max_epochs 4 --num_workers 2 --load_checkpoint True --checkpoint_path ../drive/MyDrive/LOGS/lightning_logs/version_0/checkpoints"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zwlYsjznDLJS","outputId":"1f4e6cbe-45b7-4153-d6ee-e214f21d5649","executionInfo":{"status":"ok","timestamp":1680886827834,"user_tz":-120,"elapsed":7808,"user":{"displayName":"domenico muscillo","userId":"05625114363572629197"}}},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py:554: UserWarning: This DataLoader will create 4 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n"," warnings.warn(_create_warning_msg(\n","Using 16bit None Automatic Mixed Precision (AMP)\n","GPU available: True (cuda), used: True\n","TPU available: False, using: 0 TPU cores\n","IPU available: False, using: 0 IPUs\n","HPU available: False, using: 0 HPUs\n","Traceback (most recent call last):\n"," File \"/content/Simple_VPR_codebase/main.py\", line 143, in <module>\n"," model = LightningModel.load_from_checkpoint(ckpt)\n"," File \"/usr/local/lib/python3.9/dist-packages/pytorch_lightning/core/saving.py\", line 139, in load_from_checkpoint\n"," return _load_from_checkpoint(\n"," File \"/usr/local/lib/python3.9/dist-packages/pytorch_lightning/core/saving.py\", line 188, in _load_from_checkpoint\n"," return _load_state(cls, checkpoint, strict=strict, **kwargs)\n"," File \"/usr/local/lib/python3.9/dist-packages/pytorch_lightning/core/saving.py\", line 234, in _load_state\n"," obj = cls(**_cls_kwargs)\n","TypeError: __init__() missing 2 required positional arguments: 'val_dataset' and 'test_dataset'\n"]}]}]}
0 commit comments