upload_files tests passed

Author: Sid Mohan

examples/uploading-file-types.ipynb

@@ -31,7 +31,7 @@
    "outputs": [],
    "source": [
     "# Initialize\n",
-    "%pip install datafog==2.4.0b1\n",
+    "%pip install datafog==2.4.0b3\n",
     "import json\n",
     "\n",
     "import requests\n",
@@ -122,27 +122,31 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip list"
+    "# Email confirmation for a event meetup\n",
+    "# input_file = \"/Users/sidmohan/Desktop/datafog-v2.4.0/datafog-python/tests/files/input_files/agi-builder-meetup.pdf\"\n",
+    "\n",
+    "# readthedocs for PyPDF\n",
+    "input_file = \"/Users/sidmohan/Desktop/datafog-v2.4.0/datafog-python/tests/files/input_files/pypdf-readthedocs-io-en-stable.pdf\"\n",
+    "\n",
+    "\n",
+    "# input_file = \"/Users/sidmohan/Desktop/datafog-v2.4.0/datafog-python/tests/files/input_files/pypdf-readthedocs-io-en-stable.pdf\"\n",
+    "output = datafog.DataFog.upload_file(uploaded_file_path=input_file)\n",
+    "print(output)"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 3,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "{'agi-builder-meetup.pdf': \"2/26/24, 2:16 PM\\nAGI Builders Meetup SF · Luma\\nContact the HostReport Event29\\nEvent FullIf youʼd like, you can join the waitlist.Please click on the button below to join the waitlist.You will be notified if additional spots becomeavailable.\\nSubscribe\\nHosted ByEric LIU\\n5\\x0030pm - 6\\x0000pm: Doors open and check-in.\\nRegistration\\nFEBThursday, February 295\\x0030 PM - 8\\x0000 PM\\nOpen stage for AI builders, researchersand enthusiasts to share, inspire andtransform.\\nJoin Waitlist\\x00\\x00\\x00\\x00 PM PST\\nPresented byAGI Builders M…\\nAbout Event👋 We're thrilled to invite you to the first AGI Buildersmeetup on the leap day of 2024, February 29th.❤ It's a gathering where AI builders, researchers andenthusiasts share ideas, inspire peers and transform thefuture.💡 Participants can expect engaging tech talks coveringthe latest challenges and advancements in AI.🍕 Light refreshments will be available.Agenda:\\nAGI Builders Meetup SF\\nCloudflareSan Francisco, California\\nFeatured in Generative AI San Fra…\\nSign In\\nhttps://lu.ma/32549yyf\\n1/3\\n2/26/24, 2:16 PM\\nAGI Builders Meetup SF · Luma\\n7\\x0040pm - 8\\x0000pm: NetworkingAbout the hosts:Cloudflare helps organizations make employees,applications and networks faster & more secure.BentoML empowers developers to run any AI models inthe cloud and scale with confidence.Note:\\nThis event will be held in person, and due to limitedcapacity, registration is required for entry.Registration will close 2 days before the event.\\n7\\x0010pm - 7\\x0040pm: Enterprise Retrieval - AugmentedGeneration with LlamaIndex\\nWe host monthly meetups in San Francisco, havean idea you'd like to present at future events?Please apply here.\\nLocationCloudflare101 Townsend St, San Francisco, CA 94107, USA\\nby Chaoyu Yang, Founder & CEO, BentoML\\n6\\x0000pm - 6\\x0010pm: Opening\\nby Laurie Voss, VP Developer Relations,LlamaIndex\\n6\\x0040pm - 7\\x0010pm: RAG as a service with BentoML\\n6\\x0010pm - 6\\x0040pm: Phoney AI\\nby Craig Dennis, Developer Educator AI,Cloudflare\\nhttps://lu.ma/32549yyf\\n2/3\\n2/26/24, 2:16 PM\\nAGI Builders Meetup SF · Luma\\nWhat's NewExplorePricingHelp\\nMap data ©2024 Google\\nView larger map\\nhttps://lu.ma/32549yyf\\n3/3\\n\"}\n"
-     ]
-    }
-   ],
    "source": [
-    "input_file = \"/Users/sidmohan/Desktop/datafog-v2.4.0/datafog-python/tests/files/input_files/agi-builder-meetup.pdf\"\n",
-    "output = datafog.DataFog.upload_file(uploaded_file_path=input_file)\n",
-    "print(output)"
+    "### PPTX \n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

requirements.txt

@@ -9,5 +9,6 @@ yarl==1.8.1
 frozenlist==1.3.1
 en_spacy_pii_fast
 unstructured[pdf]
+unstructured[pptx]
 
 

setup.py

@@ -6,7 +6,7 @@
 
 
 def __version__():
-    return "2.4.0b1"
+    return "2.4.0b4"
 
 
 project_urls = {

src/datafog/__init__.py

@@ -3,6 +3,7 @@
 import logging
 import tempfile
 from pathlib import Path
+from typing import List
 
 import pandas as pd
 import requests
@@ -32,6 +33,7 @@ class DataFog:
         nlp (spacy.lang): Spacy language model for PII detection.
     """
 
+    # Maintaining support
     def __init__(self):
         """
         Initialize the DataFog instance.
@@ -59,6 +61,7 @@ def upload_file(uploaded_file_path):
         if not uploaded_file_path.exists():
             return "File not found."
         else:
+
             temp_file = tempfile.NamedTemporaryFile(
                 delete=True, suffix=uploaded_file_path.suffix
             )
@@ -71,6 +74,23 @@ def upload_file(uploaded_file_path):
 
         return texts
 
+    @staticmethod
+    def upload_files(uploaded_files: List[str]):
+        """
+        Process uploaded files.
+
+        Args:
+            uploaded_files (List[str]): A list of file paths uploaded by the user.
+
+        Returns:
+            Dict[str, str]: A dictionary containing the processed text for each uploaded file.
+        """
+        texts = {}
+        for uploaded_file in uploaded_files:
+            result = DataFog.upload_file(uploaded_file)
+            texts.update(result)
+        return texts
+
     def __call__(self, input_source, privacy_operation):
         """
         Process the input data and apply the specified privacy operation.

tests/files/input_files/pypdf-readthedocs-io-en-stable.pdf

@@ -1,5 +1,6 @@
 # test_datafog.py
 import pytest
+import requests
 
 from datafog import DataFog
 
@@ -15,7 +16,7 @@ def test_upload_file(datafog):
     file_text = result[
         uploaded_file.split("/")[-1]
     ]  # Extract the text using the file name as key
-    assert "Cloudflare" in file_text  # confirms that OCR is not on
+    assert "Cloudflare" in file_text
     assert "SF" in file_text
     assert "Laurie" in file_text
     assert "BentoML" in file_text
@@ -24,43 +25,68 @@ def test_upload_file(datafog):
     assert "LangChain" not in file_text
 
 
-# def test_call_with_redact(datafog):
-#     url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
-#     privacy_operation = "redact"
+def test_upload_files(datafog):
+    uploaded_files = [
+        "tests/files/input_files/agi-builder-meetup.pdf",
+        "tests/files/input_files/pypdf-readthedocs-io-en-stable.pdf",
+    ]
+    result = datafog.upload_files(uploaded_files=uploaded_files)
+    assert "agi-builder-meetup.pdf" in result
+    assert "pypdf-readthedocs-io-en-stable.pdf" in result
+    assert "Cloudflare" in result["agi-builder-meetup.pdf"]
+    assert "SF" in result["agi-builder-meetup.pdf"]
+    assert "Laurie" in result["agi-builder-meetup.pdf"]
+    assert "BentoML" in result["agi-builder-meetup.pdf"]
+    assert "Llama-Index" not in result["agi-builder-meetup.pdf"]
+    assert "LlamaIndex" in result["agi-builder-meetup.pdf"]
+    assert "LangChain" not in result["agi-builder-meetup.pdf"]
+    assert "John Doe" not in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "Emily Davis" not in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "546 Birch St" not in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "Newville" not in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "PyPDF" in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "CLI" in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "Python" in result["pypdf-readthedocs-io-en-stable.pdf"]
+    assert "PyPDF2" in result["pypdf-readthedocs-io-en-stable.pdf"]
 
-#     result = datafog(url, privacy_operation)
 
-#     assert "[REDACTED]" in result
-#     assert "Joe Biden" not in result
+def test_call_with_redact(datafog):
+    url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
+    privacy_operation = "redact"
 
+    result = datafog(url, privacy_operation)
 
-# def test_call_with_annotate(datafog):
-#     file_path = "sotu_2023.txt"
-#     url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
-#     file_content = requests.get(url).text
-#     with open(file_path, "w") as file:
-#         file.write(file_content)
-#     privacy_operation = "annotate"
+    assert "[REDACTED]" in result
+    assert "Joe Biden" not in result
 
-#     result = datafog(str(file_path), privacy_operation)
 
-#     assert "[ORG]" in result
-#     assert "Joe Biden" not in result
+def test_call_with_annotate(datafog):
+    file_path = "sotu_2023.txt"
+    url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
+    file_content = requests.get(url).text
+    with open(file_path, "w") as file:
+        file.write(file_content)
+    privacy_operation = "annotate"
 
+    result = datafog(str(file_path), privacy_operation)
 
-# def test_call_with_unsupported_input_type(datafog):
-#     input_source = 123  # Invalid input type
-#     privacy_operation = "redact"
+    assert "[ORG]" in result
+    assert "Joe Biden" not in result
 
-#     with pytest.raises(ValueError, match="Unsupported input source type"):
-#         datafog(input_source, privacy_operation)
 
+def test_call_with_unsupported_input_type(datafog):
+    input_source = 123  # Invalid input type
+    privacy_operation = "redact"
 
-# def test_call_with_unsupported_privacy_operation(datafog):
-#     url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
-#     privacy_operation = "invalid_operation"
+    with pytest.raises(ValueError, match="Unsupported input source type"):
+        datafog(input_source, privacy_operation)
 
-#     with pytest.raises(
-#         ValueError, match=f"Unsupported privacy operation: {privacy_operation}"
-#     ):
-#         datafog(url, privacy_operation)
+
+def test_call_with_unsupported_privacy_operation(datafog):
+    url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
+    privacy_operation = "invalid_operation"
+
+    with pytest.raises(
+        ValueError, match=f"Unsupported privacy operation: {privacy_operation}"
+    ):
+        datafog(url, privacy_operation)

tests/test_datafog.py

@@ -1,85 +1,85 @@
-# import os
-# import sys
+import os
+import sys
 
-# import requests
+import requests
 
-# from datafog import PresidioEngine as presidio
+from datafog import PresidioEngine as presidio
 
-# sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 
-# def test_piiscan_json_detection():
-#     # Assuming sample_file already contains the data you want to test
-#     sample_file = "tests/files/input_files/sample.json"
+def test_piiscan_json_detection():
+    # Assuming sample_file already contains the data you want to test
+    sample_file = "tests/files/input_files/sample.json"
 
-#     # Read the contents of sample_file into original_value
-#     with open(sample_file, "r") as f:
-#         original_value = f.read()
+    # Read the contents of sample_file into original_value
+    with open(sample_file, "r") as f:
+        original_value = f.read()
 
-#     pii_detected = presidio.scan(original_value)
+    pii_detected = presidio.scan(original_value)
 
-#     # write pii_detected to a JSON file called output.json in the same directory as sample_file
-#     with open(
-#         "tests/files/output_files/output.json",
-#         "w",
-#     ) as f:
-#         for entity in pii_detected:
-#             f.write(entity.entity_type)
-#             f.write("\n")
+    # write pii_detected to a JSON file called output.json in the same directory as sample_file
+    with open(
+        "tests/files/output_files/output.json",
+        "w",
+    ) as f:
+        for entity in pii_detected:
+            f.write(entity.entity_type)
+            f.write("\n")
 
 
-# def test_piiscan_csv_detection():
-#     # Assuming sample_file already contains the data you want to test
-#     sample_file = "tests/files/input_files/sample.csv"
+def test_piiscan_csv_detection():
+    # Assuming sample_file already contains the data you want to test
+    sample_file = "tests/files/input_files/sample.csv"
 
-#     # Read the contents of sample_file into original_value
-#     with open(sample_file, "r") as f:
-#         original_value = f.read()
+    # Read the contents of sample_file into original_value
+    with open(sample_file, "r") as f:
+        original_value = f.read()
 
-#     pii_detected = presidio.scan(original_value)
+    pii_detected = presidio.scan(original_value)
 
-#     # write pii_detected to a JSON file called output.json in the same directory as sample_file
-#     with open(
-#         "tests/files/output_files/output.csv",
-#         "w",
-#     ) as f:
-#         for entity in pii_detected:
-#             f.write(entity.entity_type)
-#             f.write("\n")
+    # write pii_detected to a JSON file called output.json in the same directory as sample_file
+    with open(
+        "tests/files/output_files/output.csv",
+        "w",
+    ) as f:
+        for entity in pii_detected:
+            f.write(entity.entity_type)
+            f.write("\n")
 
 
-# def test_piiscan_txt_detection():
-#     # Assuming sample_file already contains the data you want to test
-#     sample_file = "tests/files/input_files/sample.txt"
-#     # Read the contents of sample_file into original_value
-#     with open(sample_file, "r") as f:
-#         original_value = f.read()
+def test_piiscan_txt_detection():
+    # Assuming sample_file already contains the data you want to test
+    sample_file = "tests/files/input_files/sample.txt"
+    # Read the contents of sample_file into original_value
+    with open(sample_file, "r") as f:
+        original_value = f.read()
 
-#     pii_detected = presidio.scan(original_value)
+    pii_detected = presidio.scan(original_value)
 
-#     # write pii_detected to a JSON file called output.json in the same directory as sample_file
-#     with open(
-#         "tests/files/output_files/output.txt",
-#         "w",
-#     ) as f:
-#         for entity in pii_detected:
-#             f.write(entity.entity_type)
-#             f.write("\n")
+    # write pii_detected to a JSON file called output.json in the same directory as sample_file
+    with open(
+        "tests/files/output_files/output.txt",
+        "w",
+    ) as f:
+        for entity in pii_detected:
+            f.write(entity.entity_type)
+            f.write("\n")
 
 
-# def test_piiscan_url_detection():
-#     # Assuming sample_file already contains the data you want to test
-#     sample_url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
+def test_piiscan_url_detection():
+    # Assuming sample_file already contains the data you want to test
+    sample_url = "https://gist.githubusercontent.com/sidmohan0/1aa3ec38b4e6594d3c34b113f2e0962d/raw/42e57146197be0f85a5901cd1dcdd9ad15b31bab/sotu_2023.txt"
 
-#     response = requests.get(sample_url)
-#     original_value = response.text
-#     pii_detected = presidio.scan(original_value)
+    response = requests.get(sample_url)
+    original_value = response.text
+    pii_detected = presidio.scan(original_value)
 
-#     # write pii_detected to a output.md in the same directory as sample_url
-#     with open(
-#         "tests/files/output_files/output.md",
-#         "w",
-#     ) as f:
-#         for entity in pii_detected:
-#             f.write(entity.entity_type)
-#             f.write("\n")
+    # write pii_detected to a output.md in the same directory as sample_url
+    with open(
+        "tests/files/output_files/output.md",
+        "w",
+    ) as f:
+        for entity in pii_detected:
+            f.write(entity.entity_type)
+            f.write("\n")