Actually pass self test when using URLs

src/toil/jobStores/abstractJobStore.py

@@ -628,15 +628,15 @@ def _url_exists(cls, url: ParseResult) -> bool:
         """
         Return True if the item at the given URL exists, and Flase otherwise.
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
     def _get_size(cls, url: ParseResult) -> Optional[int]:
         """
         Get the size of the object at the given URL, or None if it cannot be obtained.
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -650,7 +650,7 @@ def _get_is_directory(cls, url: ParseResult) -> bool:
                in the storage mechanism of a supported URL scheme e.g. a blob
                in an AWS s3 bucket.
         """
-        raise NotImplementedError
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -670,7 +670,7 @@ def _read_from_url(cls, url: ParseResult, writable: IO[bytes]) -> Tuple[int, boo
 
         :return: The size of the file in bytes and whether the executable permission bit is set
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -689,7 +689,7 @@ def _list_url(cls, url: ParseResult) -> List[str]:
 
         :return: The children of the given URL, already URL-encoded.
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -701,7 +701,7 @@ def _open_url(cls, url: ParseResult) -> IO[bytes]:
 
         Raises FileNotFoundError if the thing at the URL is not found.
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -719,7 +719,7 @@ def _write_to_url(cls, readable: Union[IO[bytes], IO[str]], url: ParseResult, ex
 
         :param bool executable: determines if the file has executable permissions
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @classmethod
     @abstractmethod
@@ -735,7 +735,7 @@ def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
 
         :return bool: returns true if the cls supports the URL
         """
-        raise NotImplementedError()
+        raise NotImplementedError(f"No implementation for {url}")
 
     @abstractmethod
     def destroy(self) -> None:
@@ -1741,6 +1741,16 @@ class JobStoreSupport(AbstractJobStore, metaclass=ABCMeta):
     def _supports_url(cls, url: ParseResult, export: bool = False) -> bool:
         return url.scheme.lower() in ('http', 'https', 'ftp') and not export
 
+    @classmethod
+    def _url_exists(cls, url: ParseResult) -> bool:
+        try:
+            # TODO: Figure out how to HEAD instead of this.
+            with cls._open_url(url):
+                return True
+        except:
+            pass
+        return False
+
     @classmethod
     @retry(
         errors=[

src/toil/wdl/wdltoil.py

@@ -567,9 +567,9 @@ def _devirtualize_filename(self, filename: str) -> str:
                 # in, not relative to the thing.
                 parent_url = urljoin(filename, ".")
                 # Turn it into a string we can make a directory for
-                dir_path = os.path.join(self._file_store.localTempDir, quote(parent_url))
+                dir_path = os.path.join(self._file_store.localTempDir, quote(parent_url, safe=''))
 
-            if not os.path.exists(parent_id):
+            if not os.path.exists(dir_path):
                 # Make sure the chosen directory exists
                 os.mkdir(dir_path)
             # And decide the file goes in it.