diff --git a/00_core.ipynb b/00_core.ipynb index 061e206..69a7db2 100644 --- a/00_core.ipynb +++ b/00_core.ipynb @@ -449,13 +449,13 @@ " if Path(local_path).exists(): shutil.rmtree(local_path)\n", "\n", " print(f\"-----Downloading or Creating Dataset\")\n", - " if check_ds_exists(f\"{username}/{title}\"): \n", - " get_dataset(local_path,f\"{username}/{title}\",force=True)\n", - " else: \n", - " mk_dataset(local_path,title,force=True)\n", + " try: get_dataset(local_path,f\"{username}/{title}\",force=True)\n", + " except Exception as ex:\n", + " if '404' in str(ex): mk_dataset(local_path,title,force=True)\n", + " else: raise ex\n", + " \n", " print(f\"-----Checking dataset version against pip\")\n", " ver_local_orig = get_local_ds_ver(lib_path,lib)\n", - " orig_ds = Path(local_path).ls().sorted()\n", "\n", " for item in local_path.ls():\n", " if item.name not in retain: \n", @@ -464,9 +464,8 @@ " get_pip_library(local_path,lib)\n", " \n", " ver_local_new = get_local_ds_ver(lib_path,lib)\n", - " new_ds = Path(local_path).ls().sorted()\n", "\n", - " if orig_ds != new_ds: \n", + " if ver_local_new != ver_local_orig: \n", " print(f\"-----Updating {lib} in Kaggle from {ver_local_orig} to {ver_local_new}\")\n", " push_dataset(local_path,ver_local_new)\n", " else: print(f\"-----Kaggle dataset already up to date {ver_local_orig} to {ver_local_new}\")\n", diff --git a/fastkaggle/_modidx.py b/fastkaggle/_modidx.py index 71f740d..67c3d17 100644 --- a/fastkaggle/_modidx.py +++ b/fastkaggle/_modidx.py @@ -1,13 +1,9 @@ # Autogenerated by nbdev -d = { 'settings': { 'allowed_cell_metadata_keys': '', - 'allowed_metadata_keys': '', - 'audience': 'Developers', +d = { 'settings': { 'audience': 'Developers', 'author': 'Jeremy Howard', 'author_email': 'info@fast.ai', - 'black_formatting': 'False', 'branch': 'master', - 'clean_ids': 'True', 'copyright': 'Jeremy Howard, 2022 onwards', 'custom_sidebar': 'False', 'description': 'Kaggling for fast kagglers!', @@ -16,7 +12,6 @@ 'doc_path': 'docs', 'git_url': 'https://github.com/fastai/fastkaggle/tree/master/', 'host': 'github', - 'jupyter_hooks': 'True', 'keywords': 'machine-learning kaggle fastai nbdev', 'language': 'English', 'lib_name': 'fastkaggle', @@ -24,7 +19,6 @@ 'license': 'apache2', 'min_python': '3.7', 'nbs_path': '.', - 'readme_nb': 'index.ipynb', 'recursive': 'False', 'requirements': 'fastcore>=1.4.5 kaggle', 'status': '2', diff --git a/fastkaggle/core.py b/fastkaggle/core.py index ec7e9d9..2a0ded9 100644 --- a/fastkaggle/core.py +++ b/fastkaggle/core.py @@ -171,13 +171,13 @@ def create_libs_datasets(libs, # List of libraries to create datasets for (ie [' if Path(local_path).exists(): shutil.rmtree(local_path) print(f"-----Downloading or Creating Dataset") - if check_ds_exists(f"{username}/{title}"): - get_dataset(local_path,f"{username}/{title}",force=True) - else: - mk_dataset(local_path,title,force=True) + try: get_dataset(local_path,f"{username}/{title}",force=True) + except Exception as ex: + if '404' in str(ex): mk_dataset(local_path,title,force=True) + else: raise ex + print(f"-----Checking dataset version against pip") ver_local_orig = get_local_ds_ver(lib_path,lib) - orig_ds = Path(local_path).ls().sorted() for item in local_path.ls(): if item.name not in retain: @@ -186,9 +186,8 @@ def create_libs_datasets(libs, # List of libraries to create datasets for (ie [' get_pip_library(local_path,lib) ver_local_new = get_local_ds_ver(lib_path,lib) - new_ds = Path(local_path).ls().sorted() - if orig_ds != new_ds: + if ver_local_new != ver_local_orig: print(f"-----Updating {lib} in Kaggle from {ver_local_orig} to {ver_local_new}") push_dataset(local_path,ver_local_new) else: print(f"-----Kaggle dataset already up to date {ver_local_orig} to {ver_local_new}") diff --git a/index.ipynb b/index.ipynb index 155ea34..3b45d48 100644 --- a/index.ipynb +++ b/index.ipynb @@ -144,7 +144,7 @@ "metadata": {}, "outputs": [], "source": [ - "lib_path = Path('/root/kaggle_datasets')\n", + "lib_path = Path.home()/'kaggle_datasets'\n", "username = 'isaacflath'" ] }, @@ -166,20 +166,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Processing fastcore as library-fastcore at /root/kaggle_datasets/library-fastcore\n", + "Processing fastcore as library-fastcore at /Users/isaacflath/kaggle_datasets/library-fastcore\n", "-----Downloading or Creating Dataset\n", "-----Checking dataset version against pip\n", "-----Kaggle dataset already up to date 1.5.16 to 1.5.16\n", - "Processing timm as library-timm at /root/kaggle_datasets/library-timm\n", + "Processing flask as library-flask at /Users/isaacflath/kaggle_datasets/library-flask\n", "-----Downloading or Creating Dataset\n", "-----Checking dataset version against pip\n", - "-----Kaggle dataset already up to date 0.6.7 to 0.6.7\n", + "-----Kaggle dataset already up to date 2.2.2 to 2.2.2\n", + "Processing fastkaggle as library-fastkaggle at /Users/isaacflath/kaggle_datasets/library-fastkaggle\n", + "-----Downloading or Creating Dataset\n", + "-----Checking dataset version against pip\n", + "-----Kaggle dataset already up to date 0.0.6 to 0.0.6\n", "Complete\n" ] } ], "source": [ - "libs = ['fastcore','timm']\n", + "libs = ['fastcore','flask','fastkaggle']\n", "create_libs_datasets(libs,lib_path,username)" ] },