Add compose v5 compatibility, legacy-field warnings, UI error fixes

deploy/docker/.llm.env.example

@@ -1,11 +1,3 @@
-# REQUIRED for a reachable server: API token for the Docker server (0.9.0+).
-# Without it the server binds loopback inside the container and the published
-# port answers with "connection reset". Any non-empty value works, but treat it
-# as a password — use a long random string (e.g. from: openssl rand -hex 32).
-# Note: with docker compose, the token MUST be set here — exporting it in your
-# shell does not reach the container.
-CRAWL4AI_API_TOKEN=
-
 # Optional: enable declarative hooks support (disabled by default)
 # CRAWL4AI_HOOKS_ENABLED=true
 

deploy/docker/MIGRATION.md

@@ -25,12 +25,13 @@ loopback by default and will not expose itself without a credential.
 export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)"
 ```
 
-> ⚠️ **Docker Compose users:** `export` alone does **not** work — the shipped
-> `docker-compose.yml` does not forward host environment variables. Set the
-> token in the `.llm.env` file at the project root instead (the example file
-> ships an empty `CRAWL4AI_API_TOKEN=` line — fill it in).
->
-> For plain `docker run`, pass it explicitly:
+With `docker compose`, run the export in the same shell before
+`docker compose up`; the compose file passes the token into the container.
+For a persistent setup, put the `CRAWL4AI_API_TOKEN=...` line in a `.env`
+file in the project root instead — compose reads it automatically (a shell
+export still takes precedence).
+
+> ⚠️ For plain `docker run`, pass it explicitly:
 > `-e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN"` (the value-less shorthand
 > `-e CRAWL4AI_API_TOKEN` silently passes empty from a shell where the variable
 > isn't set).

deploy/docker/entrypoint.sh

@@ -31,6 +31,8 @@ else
     # No credential -> refuse to expose; serve loopback only.
     GUNICORN_BIND="127.0.0.1:${PORT}"
     echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2
+    echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2
+    echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or 'export CRAWL4AI_API_TOKEN=...' before 'docker compose up'), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2
 fi
 export GUNICORN_BIND
 

deploy/docker/schemas.py

@@ -42,6 +42,14 @@ class HookConfig(BaseModel):
         le=120,
         description="Timeout in seconds for each hook execution",
     )
+    # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an
+    # exec()-based RCE surface). Captured here (instead of being dropped by
+    # pydantic) solely so the server can tell the caller it was NOT executed;
+    # it is never run.
+    code: Optional[Any] = Field(
+        default=None,
+        description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",
+    )
 
     class Config:
         json_schema_extra = {
@@ -84,14 +92,29 @@ class ScreenshotRequest(BaseModel):
     url: str
     screenshot_wait_for: Optional[float] = 2
     wait_for_images: Optional[bool] = False
-    # output_path removed: callers never name a filesystem path (it was an
-    # arbitrary-write -> RCE vector). The server writes to the sandboxed
-    # artifact store and returns an opaque artifact_id.
+    # Deprecated no-op: caller paths were an arbitrary-write -> RCE vector.
+    # Never written; the server stores results in the artifact store instead.
+    output_path: Optional[str] = Field(
+        default=None,
+        deprecated=True,
+        description=(
+            "REMOVED in 0.9.0 and ignored - no file is written. Results are "
+            "stored server-side; fetch via GET /artifacts/{artifact_id}."
+        ),
+    )
 
 
 class PDFRequest(BaseModel):
     url: str
-    # output_path removed (see ScreenshotRequest).
+    # output_path deprecated no-op (see ScreenshotRequest).
+    output_path: Optional[str] = Field(
+        default=None,
+        deprecated=True,
+        description=(
+            "REMOVED in 0.9.0 and ignored - no file is written. Results are "
+            "stored server-side; fetch via GET /artifacts/{artifact_id}."
+        ),
+    )
 
 
 class JSEndpointRequest(BaseModel):

deploy/docker/server.py

@@ -392,7 +392,7 @@ def _current_api_token() -> str:
 app.add_middleware(
     AuthGateMiddleware,
     token_provider=_current_api_token,
-    public_paths={HEALTH_PATH, "/token"},
+    public_paths={HEALTH_PATH, "/token", "/"},
     public_prefixes=_UI_PREFIXES,
 )
 
@@ -660,6 +660,32 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)):
 
 # Screenshot endpoint
 
+_OUTPUT_PATH_WARNING = (
+    "output_path was removed in 0.9.0 and is ignored - no file was written. "
+    "The result is stored server-side; fetch it with an authenticated "
+    "GET /artifacts/{artifact_id}."
+)
+
+_HOOKS_CODE_WARNING = (
+    "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. "
+    "Use declarative hook actions instead (GET /hooks/info for the schema)."
+)
+
+
+def _reject_disabled_hooks(hooks) -> None:
+    """403 for any hooks payload while hooks are disabled. Legacy inline code
+    gets its own detail: enabling CRAWL4AI_HOOKS_ENABLED would not run it (the
+    feature was removed in 0.9.0), so the generic remedy would mislead."""
+    if hooks.code:
+        raise HTTPException(
+            403,
+            "Inline hook code (hooks.code) was removed in 0.9.0 and cannot be "
+            "enabled; it was not executed. Use declarative hook actions instead "
+            "(GET /hooks/info), which are additionally disabled on this server "
+            "(CRAWL4AI_HOOKS_ENABLED).",
+        )
+    raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
+
 
 @app.post("/screenshot")
 @limiter.limit(config["rate_limiting"]["default_limit"])
@@ -675,6 +701,9 @@ async def generate_screenshot(
     sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it.
     """
     validate_url_scheme(body.url)
+    # model_dump, not attribute access: the field is marked deprecated and
+    # reading the attribute emits DeprecationWarning on every request.
+    legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
     crawler = None
     try:
         cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for, wait_for_images=body.wait_for_images)
@@ -684,7 +713,10 @@ async def generate_screenshot(
             raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
         screenshot_data = results[0].screenshot
         art = _store_artifact("png", base64.b64decode(screenshot_data))
-        return {"success": True, "screenshot": screenshot_data, **art}
+        response = {"success": True, "screenshot": screenshot_data, **art}
+        if legacy_output_path:
+            response["warning"] = _OUTPUT_PATH_WARNING
+        return response
     except HTTPException:
         raise
     except Exception as e:
@@ -710,6 +742,9 @@ async def generate_pdf(
     sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it.
     """
     validate_url_scheme(body.url)
+    # model_dump, not attribute access: the field is marked deprecated and
+    # reading the attribute emits DeprecationWarning on every request.
+    legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
     crawler = None
     try:
         cfg = CrawlerRunConfig(pdf=True)
@@ -719,7 +754,10 @@ async def generate_pdf(
             raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
         pdf_data = results[0].pdf
         art = _store_artifact("pdf", pdf_data)
-        return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+        response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+        if legacy_output_path:
+            response["warning"] = _OUTPUT_PATH_WARNING
+        return response
     except HTTPException:
         raise
     except Exception as e:
@@ -879,7 +917,7 @@ async def crawl(
     if not crawl_request.urls:
         raise HTTPException(400, "At least one URL required")
     if crawl_request.hooks and not HOOKS_ENABLED:
-        raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
+        _reject_disabled_hooks(crawl_request.hooks)
     # Check whether it is a redirection for a streaming request
     try:
         crawler_config = CrawlerRunConfig.load(
@@ -889,15 +927,15 @@ async def crawl(
         raise HTTPException(400, f"Rejected config: {e}")
     if crawler_config.stream:
         return await stream_process(crawl_request=crawl_request)
-    
+
     # Prepare hooks config if provided
     hooks_config = None
     if crawl_request.hooks:
         hooks_config = {
             'hooks': crawl_request.hooks.hooks,
             'timeout': crawl_request.hooks.timeout
         }
-    
+
     results = await handle_crawl_request(
         urls=crawl_request.urls,
         browser_config=crawl_request.browser_config,
@@ -906,6 +944,11 @@ async def crawl(
         hooks_config=hooks_config,
         crawler_configs=crawl_request.crawler_configs,
     )
+    if crawl_request.hooks and crawl_request.hooks.code:
+        hooks_resp = results.setdefault("hooks", {"status": "ignored", "attached": []})
+        if not crawl_request.hooks.hooks:
+            hooks_resp["status"] = "ignored"
+        hooks_resp["warning"] = _HOOKS_CODE_WARNING
     return JSONResponse(results)
 
 
@@ -919,12 +962,12 @@ async def crawl_stream(
     if not crawl_request.urls:
         raise HTTPException(400, "At least one URL required")
     if crawl_request.hooks and not HOOKS_ENABLED:
-        raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
+        _reject_disabled_hooks(crawl_request.hooks)
 
     return await stream_process(crawl_request=crawl_request)
 
 async def stream_process(crawl_request: CrawlRequestWithHooks):
-    
+
     # Prepare hooks config if provided# Prepare hooks config if provided
     hooks_config = None
     if crawl_request.hooks:
@@ -950,6 +993,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
     if hooks_info:
         import json
         headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status'])
+    if crawl_request.hooks and crawl_request.hooks.code:
+        headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNING
 
     return StreamingResponse(
         stream_results(crawler, gen),

deploy/docker/static/playground/index.html

@@ -617,6 +617,25 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
             }
         }
 
+        // Build a useful error message from a failed HTTP response
+        function httpErrorMessage(response, data) {
+            let detail = data && (data.detail || data.error);
+            // FastAPI 422s send detail as an array of error objects; other
+            // non-strings would render as [object Object].
+            if (Array.isArray(detail)) {
+                detail = detail.map(e => e && e.msg ? `${(e.loc || []).join('.')}: ${e.msg}` : JSON.stringify(e)).join('; ');
+            } else if (detail && typeof detail !== 'string') {
+                detail = JSON.stringify(detail);
+            }
+            let msg = detail || `HTTP ${response.status}`;
+            if (response.status === 401) {
+                msg += getToken()
+                    ? ' — token rejected; check the API token in the token bar (top right)'
+                    : ' — set your API token in the token bar (top right)';
+            }
+            return msg;
+        }
+
         // Generate code snippets
         function generateSnippets(api, payload, method = 'POST') {
             // Python snippet
@@ -747,11 +766,11 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                         method: 'GET',
                         headers: { 'Accept': 'application/json' }
                     });
-                    responseData = await response.json();
+                    responseData = await response.json().catch(() => ({}));
                     const time = Math.round(performance.now() - startTime);
                     if (!response.ok) {
                         updateStatus('error', time);
-                        throw new Error(responseData.error || 'Request failed');
+                        throw new Error(httpErrorMessage(response, responseData));
                     }
                     updateStatus('success', time);
                     document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);
@@ -765,6 +784,12 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                         body: JSON.stringify(payload)
                     });
 
+                    if (!response.ok) {
+                        const errData = await response.json().catch(() => ({}));
+                        updateStatus('error', Math.round(performance.now() - startTime));
+                        throw new Error(httpErrorMessage(response, errData));
+                    }
+
                     const reader = response.body.getReader();
                     let text = '';
                     let maxMemory = 0;
@@ -804,12 +829,12 @@ <h2 class="font-medium text-accent">🔥 Stress Test</h2>
                         body: JSON.stringify(payload)
                     });
 
-                    responseData = await response.json();
+                    responseData = await response.json().catch(() => ({}));
                     const time = Math.round(performance.now() - startTime);
 
                     if (!response.ok) {
                         updateStatus('error', time);
-                        throw new Error(responseData.error || 'Request failed');
+                        throw new Error(httpErrorMessage(response, responseData));
                     }
 
                     updateStatus(

deploy/docker/tests/requirements.txt

@@ -1,2 +1,3 @@
 httpx>=0.25.0
 docker>=7.0.0
+pyyaml>=6.0