Add shared grid math and local fine grid; expand PR template

Author: TayDa64

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,97 @@
+# Precision Grounding + Inspect Overlay (Opus Execution Plan)
+
+## Summary
+- Align grid math across overlay, main, and AI prompts using shared constants.
+- Add local fine grid around the cursor for precise targeting without full-grid noise.
+- Introduce devtools-style inspect overlays (actionable element boxes + metadata).
+- Ensure AI uses the same visual grounding as the user.
+
+## Goals / Non-Goals
+**Goals**
+- User and AI see the same targeting primitives (grid + inspect metadata).
+- Fine precision selection without needing full fine-grid visibility.
+- Deterministic coordinate mapping across renderer/main/AI prompt.
+
+**Non-Goals**
+- Full external app DOM access (we rely on OCR + visual detection).
+- Replacing the grid system entirely.
+
+## Problem
+- Fine dots do not appear around the cursor, preventing high-precision selection.
+- AI coordinate grounding drifts due to mismatched math across modules.
+- AI lacks the same visualization/inspection context the user sees.
+
+## Approach
+1) Shared grid math module used by renderer, main, and AI prompt.
+2) Local fine-grid rendering around cursor in selection mode.
+3) Inspect layer backed by visual-awareness to surface actionable regions.
+4) AI prompt + action executor aligned to overlay math and inspect metadata.
+
+## Key Changes (Planned)
+- `src/shared/grid-math.js`: canonical grid constants + label → pixel conversion.
+- `src/renderer/overlay/overlay.js`: local fine-grid render + shared math usage.
+- `src/renderer/overlay/preload.js`: expose grid math to renderer safely.
+- `src/main/system-automation.js`: unify coordinate mapping.
+- `src/main/ai-service.js`: ground prompts + fine label support.
+- `src/main/index.js`: optional inspect toggle + overlay commands.
+- `src/main/visual-awareness.js`: actionable element detection + metadata surface.
+
+## Implementation Plan
+**Phase 1: Grounding & Precision**
+- [x] Shared grid math module and renderer/main integration.
+- [x] Local fine-grid around cursor with snap highlight.
+- [ ] Add label→pixel IPC from main to overlay to guarantee exact mapping.
+- [ ] Add fine label rendering on hover (C3.12) in local grid.
+
+**Phase 2: Inspect Overlay (Devtools‑Style)**
+- [ ] Add inspect toggle command and UI indicator.
+- [ ] Visual-awareness pass: actionable region detection (buttons, inputs, links).
+- [ ] Overlay layer draws bounding boxes + tooltip with text/role/confidence.
+- [ ] Selection handoff: click through to element center.
+
+**Phase 3: AI Grounding + Action Execution**
+- [ ] Include inspect metadata + screen size in AI context.
+- [ ] Prefer inspect targets; fallback to grid only if needed.
+- [ ] Add “precision click” action with safety confirmation.
+
+## UX Notes
+- Inspect mode should be visually distinct (e.g., cyan boxes, tooltip anchored).
+- Local fine grid should fade in/out smoothly and never block click-through.
+- Keep overlays under 16ms frame budget; throttle redraw to pointer move.
+
+## Testing
+**Unit**
+- Grid label conversions (coarse + fine).
+- Shared constants remain consistent across renderer/main/AI.
+
+**Manual**
+- Cursor-local fine dots appear in selection mode and track cursor.
+- Background click-through still works in both modes.
+- Inspect overlay alignment with visible UI elements.
+
+**Regression**
+- Coarse grid rendering.
+- Pulse effect visibility.
+- Safety confirmation flow intact.
+
+## Risks / Mitigations
+- DPI scaling drift → use Electron `screen.getPrimaryDisplay().scaleFactor`.
+- Performance → local fine grid only; throttled draw.
+- Overlay click-through → hide overlay only at click execution.
+
+## Observability / Debugging
+- Add a debug overlay toggle for grid math readouts.
+- Log label→pixel conversions when in inspect mode.
+- Capture last 10 action targets in memory for post-mortem.
+
+## Opus Notes (Websearch Required)
+- Verify Electron overlay best practices (`setIgnoreMouseEvents` behavior).
+- Validate DPI/scaling guidance for Windows and macOS.
+- Check common patterns for devtools-like overlays.
+
+## Checklist
+- [ ] Shared grid math used everywhere (renderer, main, AI prompt).
+- [ ] Local fine grid visible and performant.
+- [ ] Inspect overlay works and aligns with AI context.
+- [ ] AI actions target inspect regions with correct coordinates.
+- [ ] Tests updated/added and passing.

package.json

@@ -4,7 +4,8 @@
   "description": "GitHub Copilot CLI with headless agent + ultra-thin overlay architecture",
   "main": "src/main/index.js",
   "scripts": {
-    "start": "node scripts/start.js"
+    "start": "node scripts/start.js",
+    "test": "node scripts/test-grid.js"
   },
   "keywords": [
     "copilot",

scripts/test-grid.js

@@ -0,0 +1,18 @@
+const assert = require('assert');
+const { gridToPixels } = require('../src/main/system-automation');
+
+function expectCoord(label, expectedX, expectedY) {
+  const result = gridToPixels(label);
+  assert.strictEqual(result.x, expectedX, `${label} x`);
+  assert.strictEqual(result.y, expectedY, `${label} y`);
+}
+
+expectCoord('A0', 50, 50);
+expectCoord('B0', 150, 50);
+expectCoord('A1', 50, 150);
+expectCoord('C3', 250, 350);
+expectCoord('Z0', 2550, 50);
+expectCoord('AA0', 2650, 50);
+expectCoord('C3.12', 237.5, 362.5);
+
+console.log('gridToPixels tests passed');

src/main/ai-service.js

@@ -103,6 +103,7 @@ const SYSTEM_PROMPT = `You are Liku, an intelligent AGENTIC AI assistant integra
    - **Format**: "C3" = column C (index 2), row 3 = pixel (250, 350)
    - **Formula**: x = 50 + col_index * 100, y = 50 + row_index * 100
    - A0 ≈ (50, 50), B0 ≈ (150, 50), A1 ≈ (50, 150)
+   - **Fine Grid**: Sub-labels like C3.12 refer to 25px subcells inside C3
 
 3. **SYSTEM CONTROL - AGENTIC ACTIONS**: You can execute actions on the user's computer:
    - **Click**: Click at coordinates
@@ -139,10 +140,11 @@ When the user asks you to DO something (click, type, interact), respond with a J
 - \`{"type": "screenshot"}\` - Take screenshot to verify result
 
 ### Grid to Pixel Conversion:
-- A1 → (100, 100), B1 → (200, 100), C1 → (300, 100)
-- A2 → (100, 200), B2 → (200, 200), C2 → (300, 200)
-- Formula: x = 100 + (column_number - 1) * 100, y = 100 + (row_number - 1) * 100
-- Column A=1, B=2, C=3... so C3 = x: 100 + 2*100 = 300, y: 100 + 2*100 = 300
+- A0 → (50, 50), B0 → (150, 50), C0 → (250, 50)
+- A1 → (50, 150), B1 → (150, 150), C1 → (250, 150)
+- Formula: x = 50 + col_index * 100, y = 50 + row_index * 100
+- Column A=0, B=1, C=2... so C3 = x: 50 + 2*100 = 250, y: 50 + 3*100 = 350
+ - Fine labels: C3.12 = x: 12.5 + (2*4+1)*25 = 237.5, y: 12.5 + (3*4+2)*25 = 362.5
 
 ## Response Guidelines
 
@@ -1519,7 +1521,7 @@ async function resumeAfterConfirmation(onAction = null, onScreenshot = null, opt
  * Convert grid coordinate to pixel position
  */
 function gridToPixels(coord) {
-  return systemAutomation.gridToPixels(coord, { width: 1920, height: 1080 });
+  return systemAutomation.gridToPixels(coord);
 }
 
 module.exports = {

src/main/system-automation.js

@@ -7,6 +7,7 @@
 
 const { exec, spawn } = require('child_process');
 const path = require('path');
+const gridMath = require('../shared/grid-math');
 
 // Action types the AI can request
 const ACTION_TYPES = {
@@ -495,32 +496,18 @@ function parseAIActions(aiResponse) {
  * @param {Object} screenSize - {width, height} of the screen
  * @param {number} coarseSpacing - Spacing of coarse grid (default 100)
  */
-function gridToPixels(coord, screenSize, coarseSpacing = 100) {
-  // Parse coordinate: letters for column, numbers for row
-  const match = coord.match(/^([A-Za-z]+)(\d+)$/);
-  if (!match) {
+function gridToPixels(coord) {
+  const coords = gridMath.labelToScreenCoordinates(coord);
+  if (!coords) {
     throw new Error(`Invalid coordinate format: ${coord}`);
   }
-  
-  const colStr = match[1].toUpperCase();
-  const row = parseInt(match[2], 10);
-  
-  // Convert column letters to number (A=0, B=1, ..., Z=25, AA=26, etc.)
-  let col = 0;
-  for (let i = 0; i < colStr.length; i++) {
-    col = col * 26 + (colStr.charCodeAt(i) - 64);
-  }
-  col--; // Make 0-indexed
-  
-  // Calculate pixel position - grid starts at startOffset (50px) to cover full screen
-  // This MUST match overlay.js: startOffset = coarseSpacing / 2
-  const startOffset = coarseSpacing / 2; // 50px for default 100px spacing
-  const x = startOffset + col * coarseSpacing;
-  const y = startOffset + row * coarseSpacing;
-  
-  console.log(`[AUTOMATION] gridToPixels: ${coord} -> col=${col}, row=${row} -> (${x}, ${y})`);
-  
-  return { x, y, col, row };
+
+  const labelInfo = coords.isFine
+    ? `fineCol=${coords.fineCol}, fineRow=${coords.fineRow}`
+    : `col=${coords.colIndex}, row=${coords.rowIndex}`;
+  console.log(`[AUTOMATION] gridToPixels: ${coord} -> ${labelInfo} -> (${coords.x}, ${coords.y})`);
+
+  return coords;
 }
 
 module.exports = {