Created using Colab

Author: alexfazio

Diff for: devdocs_to_llm_firecrawl.ipynb

@@ -5,7 +5,7 @@
     "colab": {
       "private_outputs": true,
       "provenance": [],
-      "authorship_tag": "ABX9TyNVJFk3jg83ovhaMDcJHOSo",
+      "authorship_tag": "ABX9TyMK76muNMFRt0JTXy2fMPCy",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -290,11 +290,18 @@
       "execution_count": null,
       "outputs": []
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Crawl Launch"
+      ],
+      "metadata": {
+        "id": "lTFC4NkOqxUd"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
-        "# @title Crawl Launch\n",
-        "\n",
         "import requests\n",
         "from bs4 import BeautifulSoup\n",
         "from urllib.parse import urljoin, urlparse\n",
@@ -421,11 +428,18 @@
         "id": "ce2N8HasNylT"
       }
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Scraping Options"
+      ],
+      "metadata": {
+        "id": "HZau_9Hsq1Jg"
+      }
+    },
     {
       "cell_type": "code",
       "source": [
-        "# @title Scraping Options\n",
-        "\n",
         "# Create a dropdown for scrape options\n",
         "scrape_option = \"Specific number of pages\"  # @param [\"All pages\", \"Specific number of pages\"]\n",
         "\n",
@@ -486,15 +500,16 @@
     },
     {
       "cell_type": "markdown",
-      "source": [],
+      "source": [
+        "# Rate Limiting and Retry Parameters\n"
+      ],
       "metadata": {
         "id": "2fqRWptDfhPc"
       }
     },
     {
       "cell_type": "code",
       "source": [
-        "# @title Rate Limiting and Retry Parameters\n",
         "# @markdown Set the rate limiting and retry parameters for the web scraper:\n",
         "\n",
         "pages_per_minute = 9 # @param {type:\"integer\"}\n",