Scene screenshots (#134)

* cache images per scene

* save token by scene

* bit more manageable

* feat: add thumbnail generation for scene images with max width 320px

* feat: add thumbnail generation to scene image upsert in frames API

* feat: add thumbnail generation to SceneImage upsert logic

* refactor: restructure scene panel layout for improved readability

* feat: update FrameImage to use thumbnail instead of full image

* fix: remove thumb suffix from subEntityId for correct route loading

* feat: load thumbnail in FrameImage when thumb prop is true

* store thumb, send header

* feat: publish new scene image via websocket on frame image retrieval

* fix: update publish_message to use nested data structure for frame info

* refresh scene image

* scene images

* 320x320

* fix test

* better placeholders

* declutter

* image aspect

Author: mariusandra

.gitignore

@@ -37,3 +37,4 @@ e2e/tmp/
 
 gpt.txt
 .env
+.aider*

backend/app/api/__init__.py

@@ -21,6 +21,7 @@
 from .fonts import *  # noqa: E402, F403
 from .log import *  # noqa: E402, F403
 from .repositories import *  # noqa: E402, F403
+from .scene_images import *  # noqa: E402, F403
 from .settings import *  # noqa: E402, F403
 from .ssh import *  # noqa: E402, F403
 from .templates import *  # noqa: E402, F403

backend/app/api/frames.py

@@ -29,6 +29,7 @@
 from app.config import config
 from app.utils.network import is_safe_host
 from app.redis import get_redis
+from app.websockets import publish_message
 from . import api_with_auth, api_no_auth
 
 
@@ -56,8 +57,8 @@ async def api_frame_get_logs(id: int, db: Session = Depends(get_db)):
     return {"logs": logs}
 
 
-@api_with_auth.get("/frames/{id:int}/image_link", response_model=FrameImageLinkResponse)
-async def get_image_link(id: int):
+@api_with_auth.get("/frames/{id:int}/image_token", response_model=FrameImageLinkResponse)
+async def get_image_token(id: int):
     expire_minutes = 5
     now = datetime.utcnow()
     expire = now + timedelta(minutes=expire_minutes)
@@ -67,7 +68,7 @@ async def get_image_link(id: int):
     expires_in = int((expire - now).total_seconds())
 
     return {
-        "url": config.ingress_path + f"/api/frames/{id}/image?token={token}",
+        "token": token,
         "expires_in": expires_in
     }
 
@@ -108,6 +109,57 @@ async def api_frame_get_image(
 
         if response.status_code == 200:
             await redis.set(cache_key, response.content, ex=86400 * 30)
+            scene_id = response.headers.get('x-scene-id')
+            if not scene_id:
+                scene_id = await redis.get(f"frame:{id}:active_scene")
+                if scene_id:
+                    scene_id = scene_id.decode('utf-8')
+            if scene_id:
+                # dimensions (best‑effort – don’t crash if Pillow missing)
+                width = height = None
+                try:
+                    from PIL import Image
+                    import io
+                    img = Image.open(io.BytesIO(response.content))
+                    width, height = img.width, img.height
+                except Exception:
+                    pass
+
+                # upsert into SceneImage
+                from app.models.scene_image import SceneImage
+                from app.api.scene_images import _generate_thumbnail
+                now = datetime.utcnow()
+                img_row = (
+                    db.query(SceneImage)
+                      .filter_by(frame_id=id, scene_id=scene_id)
+                      .first()
+                )
+                thumb, t_width, t_height = _generate_thumbnail(response.content)
+                if img_row:
+                    img_row.image      = response.content
+                    img_row.timestamp  = now
+                    img_row.width      = width
+                    img_row.height     = height
+                    img_row.thumb_image = thumb
+                    img_row.thumb_width = t_width
+                    img_row.thumb_height = t_height
+                else:
+                    img_row = SceneImage(
+                        frame_id    = id,
+                        scene_id    = scene_id,
+                        image       = response.content,
+                        timestamp   = now,
+                        width       = width,
+                        height      = height,
+                        thumb_image = thumb,
+                        thumb_width = t_width,
+                        thumb_height = t_height
+                    )
+                    db.add(img_row)
+                db.commit()
+
+            await publish_message(redis, "new_scene_image", {"frameId": id, "sceneId": scene_id, "timestamp": now.isoformat(), "width": width, "height": height})
+
             return Response(content=response.content, media_type='image/png')
         else:
             raise HTTPException(status_code=response.status_code, detail="Unable to fetch image")
@@ -142,7 +194,10 @@ async def api_frame_get_state(id: int, db: Session = Depends(get_db), redis: Red
 
         if response.status_code == 200:
             await redis.set(cache_key, response.content, ex=1)
-            return response.json()
+            state = response.json()
+            if state.get('sceneId'):
+                await redis.set(f"frame:{frame.id}:active_scene", state.get('sceneId'))
+            return state
         else:
             last_state = await redis.get(cache_key)
             if last_state:
@@ -178,7 +233,10 @@ async def api_frame_get_states(id: int, db: Session = Depends(get_db), redis: Re
 
         if response.status_code == 200:
             await redis.set(cache_key, response.content, ex=1)
-            return response.json()
+            states = response.json()
+            if states.get('sceneId'):
+                await redis.set(f"frame:{frame.id}:active_scene", states.get('sceneId'))
+            return states
         else:
             last_states = await redis.get(cache_key)
             if last_states:

backend/app/api/scene_images.py

@@ -0,0 +1,150 @@
+# backend/app/api/scene_images.py
+import io
+from jose import JWTError, jwt
+
+from fastapi import Depends, HTTPException, Request
+from fastapi.responses import StreamingResponse
+from PIL import Image, ImageDraw, ImageFont
+from sqlalchemy.orm import Session
+from app.api.auth import ALGORITHM, SECRET_KEY
+
+from app.config import config
+from app.database import get_db
+from app.models.scene_image import SceneImage            # created earlier
+from app.models.frame import Frame
+from . import api_no_auth
+
+
+def _generate_placeholder(
+    width: int | None = 320,
+    height: int | None = 240,
+    *,
+    font_path: str = "../frameos/assets/compiled/fonts/Ubuntu-Regular.ttf",
+    font_size: int = 32,
+    message: str = "No snapshot",
+) -> bytes:
+    """
+    Return a PNG (bytes) that shows a black rectangle with centred white text.
+
+    Parameters
+    ----------
+    width, height  :  Dimensions in pixels; defaults are 400×300.
+    font_path      :  Path to a scalable font file (TTF/OTF).  If omitted,
+                      Pillow’s 8‑pixel bitmap font is used.
+    font_size      :  Point size for the scalable font.
+    message        :  The text to write.
+    """
+    width, height = int(width or 320), int(height or 240)
+
+    img = Image.new("RGB", (width, height), "#1f2937")
+    draw = ImageDraw.Draw(img)
+    font = ImageFont.truetype(font_path, font_size)
+
+    left, top, right, bottom = draw.textbbox((0, 0), message, font=font)
+    text_w, text_h = right - left, bottom - top
+
+    draw.text(
+        ((width - text_w) / 2, (height - text_h) / 2),
+        message,
+        fill="white",
+        font=font,
+    )
+
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    buf.seek(0)
+    return buf.read()
+
+def _generate_thumbnail(image_bytes: bytes) -> tuple[bytes, int, int]:
+    """
+    Generate a JPEG thumbnail whose width and height never exceed 320px,
+    while preserving aspect ratio.
+    Returns (jpeg_bytes, new_width, new_height).
+    """
+    with Image.open(io.BytesIO(image_bytes)) as img:
+        # ensure RGB
+        if img.mode != "RGB":
+            img = img.convert("RGB")
+
+        orig_width, orig_height = img.size
+
+        # scale factor that keeps both sides ≤ 320
+        scale = min(320 / orig_width, 320 / orig_height, 1.0)
+        new_width  = int(round(orig_width  * scale))
+        new_height = int(round(orig_height * scale))
+
+        img = img.resize((new_width, new_height), Image.ANTIALIAS)
+
+        buf = io.BytesIO()
+        img.save(buf, format="JPEG")
+        buf.seek(0)
+        return buf.read(), new_width, new_height
+
+
+@api_no_auth.get("/frames/{frame_id}/scene_images/{scene_id}")
+async def get_scene_image(
+    frame_id: int,
+    scene_id: str,
+    token: str,
+    request: Request,
+    db: Session = Depends(get_db),
+):
+    """
+    Fetch the latest stored SceneImage.
+    If none exists, return a placeholder that matches the frame’s native
+    width/height so the UI keeps its layout.
+    """
+
+    if config.HASSIO_RUN_MODE != 'ingress':
+        try:
+            payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+            if payload.get("sub") != f"frame={frame_id}":
+                raise HTTPException(status_code=401, detail="Unauthorized")
+        except JWTError:
+            raise HTTPException(status_code=401, detail="Unauthorized")
+
+
+    img_row: SceneImage | None = (
+        db.query(SceneImage)
+        .filter_by(frame_id=frame_id, scene_id=scene_id)
+        .order_by(SceneImage.timestamp.desc())
+        .first()
+    )
+
+    if img_row:
+        # fresh snapshot found, generate and save thumbnail if not present
+        if not getattr(img_row, 'thumb_image', None):
+            thumb, t_width, t_height = _generate_thumbnail(img_row.image)
+            img_row.thumb_image = thumb
+            img_row.thumb_width = t_width
+            img_row.thumb_height = t_height
+            db.add(img_row)
+            db.commit()
+            db.refresh(img_row)
+        if request.query_params.get("thumb") == "1":
+            return StreamingResponse(
+                io.BytesIO(img_row.thumb_image),
+                media_type="image/jpeg",
+                headers={"Cache-Control": "no-cache"},
+            )
+        else:
+            return StreamingResponse(
+                io.BytesIO(img_row.image),
+                media_type="image/png",
+                headers={"Cache-Control": "no-cache"},
+            )
+
+    frame: Frame | None = db.get(Frame, frame_id)
+    if frame is None:
+        raise HTTPException(status_code=404, detail="Frame not found")
+
+    if request.query_params.get("thumb") == "1":
+        scale = min(320 / frame.width, 320 / frame.height, 1.0)
+        new_width  = int(round(frame.width  * scale))
+        new_height = int(round(frame.height * scale))
+    else:
+        new_width  = frame.width
+        new_height = frame.height
+
+    png = _generate_placeholder(new_width, new_height)
+    return StreamingResponse(io.BytesIO(png), media_type="image/png")

backend/app/api/templates.py

@@ -239,8 +239,8 @@ async def get_template(template_id: str, db: Session = Depends(get_db)):
     d = template.to_dict()
     return d
 
-@api_with_auth.get("/templates/{template_id}/image_link", response_model=TemplateImageLinkResponse)
-async def get_image_link(template_id: str):
+@api_with_auth.get("/templates/{template_id}/image_token", response_model=TemplateImageLinkResponse)
+async def get_image_token(template_id: str):
     expire_minutes = 5
     now = datetime.utcnow()
     expire = now + timedelta(minutes=expire_minutes)
@@ -249,7 +249,7 @@ async def get_image_link(template_id: str):
     expires_in = int((expire - now).total_seconds())
 
     return {
-        "url": config.ingress_path + f"/api/templates/{template_id}/image?token={token}",
+        "token": token,
         "expires_in": expires_in
     }
 

backend/app/api/tests/test_frames.py

@@ -44,10 +44,11 @@ async def test_api_frame_get_image_cached(async_client, db, redis):
     await redis.set(cache_key, b'cached_image_data')
 
     # First, get the image link (which gives us the token)
-    image_link_resp = await async_client.get(f'/api/frames/{frame.id}/image_link')
-    assert image_link_resp.status_code == 200
-    link_info = image_link_resp.json()
-    image_url = link_info['url']
+    image_token_resp = await async_client.get(f'/api/frames/{frame.id}/image_token')
+    assert image_token_resp.status_code == 200
+    link_info = image_token_resp.json()
+    token = link_info['token']
+    image_url = f'/api/frames/{frame.id}/image?token={token}'
 
     # Append t=-1 to force returning the cached data
     image_url += "&t=-1"

backend/app/models/__init__.py

@@ -5,5 +5,6 @@
 from .metrics import *  # noqa: F403
 from .repository import *  # noqa: F403
 from .settings import *  # noqa: F403
+from .scene_image import *    # noqa: F403
 from .template import *  # noqa: F403
 from .user import *  # noqa: F403

backend/app/models/log.py

@@ -61,8 +61,14 @@ async def process_log(db: Session, redis: Redis, frame: Frame, log: dict | list)
 
     assert isinstance(log, dict), f"Log must be a dict, got {type(log)}"
 
-    changes: dict[str, Any] = {}
     event = log.get('event', 'log')
+
+    if event in ("render:scene", "render:sceneChange", "event:setCurrentScene"):
+        scene_id = log.get("sceneId") or log.get("scene") or log.get("id")
+        if scene_id:
+            await redis.set(f"frame:{frame.id}:active_scene", scene_id, ex=300)
+
+    changes: dict[str, Any] = {}
     if event == 'render':
         changes['status'] = 'preparing'
     if event == 'render:device':

backend/app/models/scene_image.py

@@ -0,0 +1,38 @@
+import uuid
+from sqlalchemy import Integer, String, LargeBinary, DateTime, ForeignKey, UniqueConstraint, func
+from sqlalchemy.orm import relationship, backref, mapped_column
+from app.database import Base
+
+class SceneImage(Base):
+    """
+    Stores the *latest* rendered image for every (frame_id, scene_id) pair.
+    The row is *up‑serted* whenever a fresher snapshot is available.
+    """
+    __tablename__   = "scene_image"
+    __table_args__  = (UniqueConstraint("frame_id", "scene_id", name="u_frame_scene"),)
+
+    id        = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
+    timestamp = mapped_column(DateTime, nullable=False, default=func.current_timestamp())
+    frame_id  = mapped_column(Integer, ForeignKey("frame.id"), nullable=False)
+    scene_id  = mapped_column(String(128),               nullable=False)
+
+    image     = mapped_column(LargeBinary, nullable=False)
+    width     = mapped_column(Integer)
+    height    = mapped_column(Integer)
+
+    thumb_image     = mapped_column(LargeBinary, nullable=True)
+    thumb_width     = mapped_column(Integer)
+    thumb_height    = mapped_column(Integer)
+
+    # handy backref – Frame.scene_images
+    frame     = relationship("Frame", backref=backref("scene_images", lazy=True))
+
+    def to_dict(self):
+        return {
+            "id":        self.id,
+            "timestamp": self.timestamp.isoformat(),
+            "frame_id":  self.frame_id,
+            "scene_id":  self.scene_id,
+            "width":     self.width,
+            "height":    self.height,
+        }

backend/app/schemas/frames.py

@@ -96,7 +96,7 @@ class FrameMetricsResponse(BaseModel):
     metrics: List[Dict[str, Any]]
 
 class FrameImageLinkResponse(BaseModel):
-    url: str
+    token: str
     expires_in: int
 
 class FrameStateResponse(RootModel):