Remove dependency on requests

Author: tristan-jl

pyproject.toml

@@ -5,7 +5,7 @@ description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
 readme = "README.md"
 license = { file = "LICENSE" }
 authors = [{ name = "Shantanu Jain" }, { email = "[email protected]" }]
-dependencies = ["regex>=2022.1.18", "requests>=2.26.0"]
+dependencies = ["regex>=2022.1.18"]
 optional-dependencies = { blobfile = ["blobfile>=2"] }
 requires-python = ">=3.9"
 

tiktoken/load.py

@@ -17,11 +17,10 @@ def read_file(blobpath: str) -> bytes:
             return f.read()
 
     # avoiding blobfile for public files helps avoid auth issues, like MFA prompts
-    import requests
-
-    resp = requests.get(blobpath)
-    resp.raise_for_status()
-    return resp.content
+    import urllib.request
+    with urllib.request.urlopen(blobpath) as response:
+        resp = response.read()
+    return resp
 
 
 def check_hash(data: bytes, expected_hash: str) -> bool: