Merge pull request #231 from cortexapps/worktree-cx-2-ai-spend-metrics

feat: AI spend solution with custom metrics, scorecard, and team plugin

Author: jeff-schnitter

cortexapps_cli/commands/backup.py

@@ -471,6 +471,47 @@ def import_relationships_file(file_info):
 
     return ("entity-relationships", len(results) - failed_count, [(fp, et, em) for rt, fp, et, em in results if et])
 
+def _import_custom_metrics(ctx, directory):
+    imported = 0
+    failed = []
+    if os.path.isdir(directory):
+        print("Processing: " + directory)
+        client = ctx.obj["client"]
+        for filename in sorted(os.listdir(directory)):
+            if not filename.endswith(".json"):
+                continue
+            file_path = os.path.join(directory, filename)
+            if not os.path.isfile(file_path):
+                continue
+            metric_key = filename[:-5]  # strip .json
+            try:
+                print("   Importing: " + filename)
+                with open(file_path) as f:
+                    data = json.load(f)
+
+                # Group flat values list by entityTag
+                grouped = {}
+                for entry in data.get("values", []):
+                    tag = entry["entityTag"]
+                    if tag not in grouped:
+                        grouped[tag] = []
+                    grouped[tag].append({
+                        "timestamp": entry["timestamp"],
+                        "value": entry["value"],
+                    })
+
+                # Call per-entity bulk endpoint once per entity
+                for entity_tag, series in grouped.items():
+                    client.post(
+                        f"api/v1/eng-intel/custom-metrics/{metric_key}/entity/{entity_tag}/bulk",
+                        data={"series": series},
+                    )
+                imported += 1
+            except Exception as e:
+                print(f"   Failed to import {filename}: {type(e).__name__} - {str(e)}")
+                failed.append((file_path, type(e).__name__, str(e)))
+    return ("custom-metrics", imported, failed)
+
 def _has_relationships(file_path):
     """Check if a catalog YAML file contains x-cortex-relationships."""
     try:
@@ -760,6 +801,7 @@ def import_tenant(
     all_stats.append(_import_entity_relationship_types(ctx, directory + "/entity-relationship-types"))
     all_stats.append(_import_catalog(ctx, directory + "/catalog"))
     all_stats.append(_import_entity_relationships(ctx, directory + "/entity-relationships"))
+    all_stats.append(_import_custom_metrics(ctx, directory + "/custom-metrics"))
     all_stats.append(_import_plugins(ctx, directory + "/plugins"))
     all_stats.append(_import_scorecards(ctx, directory + "/scorecards"))
     all_stats.append(_import_workflows(ctx, directory + "/workflows"))
@@ -811,6 +853,8 @@ def import_tenant(
             elif import_type == "entity-relationships":
                 # These need special handling - would need the relationship type
                 print(f"# Manual retry needed for entity-relationships: {file_path}")
+            elif import_type == "custom-metrics":
+                print(f"# Manual retry needed for custom-metrics: {file_path}")
             elif import_type == "plugins":
                 print(f"cortex plugins create --force -f \"{file_path}\"")
             elif import_type == "scorecards":

cortexapps_cli/commands/plugins.py

@@ -87,6 +87,8 @@ def create(
             # Remove the 'tag' attribute if it exists
             data.pop("tag", None)
             r = client.put("api/v1/plugins/" + tag, data, raw_response=True)
+        else:
+            r = client.post("api/v1/plugins", data, raw_response=True)
     else:
         r = client.post("api/v1/plugins", data, raw_response=True)
 
@@ -142,12 +144,18 @@ def get(
 def replace(
     ctx: typer.Context,
     file_input: Annotated[typer.FileText, typer.Option("--file", "-f", help="File containing contents of plugin using schema defined at https://docs.cortex.io/docs/api/create-plugin")] = None,
-    tag_or_id: str = typer.Option(..., "--tag-or-id", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity.")
+    tag_or_id: str = typer.Option(None, "--tag-or-id", "-t", help="The tag or ID of the plugin to replace. Defaults to the tag field in the file."),
 ):
     """
     Replace an existing plugin by tag
     """
 
     client = ctx.obj["client"]
-    
-    client.put("api/v1/plugins/"+ tag_or_id, data=file_input.read())
+
+    data = json.loads(file_input.read())
+    resolved = tag_or_id or data.get("tag")
+    if not resolved:
+        typer.echo("Error: --tag-or-id is required when the file does not contain a 'tag' field.")
+        raise typer.Exit(1)
+
+    client.put("api/v1/plugins/" + resolved, data=data)

cortexapps_cli/commands/solutions.py

@@ -280,6 +280,7 @@ def _collect_solution_resources(path: Path) -> dict[str, list[str]]:
         "catalog": [],
         "scorecards": [],
         "workflows": [],
+        "plugins": [],
     }
     for kind in resources:
         subdir = path / kind
@@ -336,7 +337,7 @@ def _run_uninstall(client, path: Path, yes: bool) -> None:
         return
 
     typer.echo("\nThis will remove the following resources:")
-    for kind in ("workflows", "scorecards", "catalog", "entity-relationship-types", "entity-types"):
+    for kind in ("workflows", "scorecards", "plugins", "catalog", "entity-relationship-types", "entity-types"):
         count = len(resources[kind])
         if count:
             typer.echo(f"  {kind}: {count}")
@@ -352,6 +353,7 @@ def _run_uninstall(client, path: Path, yes: bool) -> None:
     steps = [
         ("workflows",                 lambda t: f"api/v1/workflows/{t}"),
         ("scorecards",                lambda t: f"api/v1/scorecards/{t}"),
+        ("plugins",                   lambda t: f"api/v1/plugins/{t}"),
         ("catalog",                   lambda t: f"api/v1/catalog/{t}"),
         ("entity-relationship-types", lambda t: f"api/v1/relationship-types/{t}"),
         ("entity-types",              lambda t: f"api/v1/catalog/definitions/{t}"),

cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml

@@ -0,0 +1,27 @@
+name: Sync Claude AI Spend to Cortex
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"  # Every Monday at 06:00 UTC
+  workflow_dispatch:       # Allow manual runs from the Actions tab
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: pip install requests
+
+      - name: Sync Claude spend to Cortex
+        env:
+          ANTHROPIC_ANALYTICS_KEY: ${{ secrets.ANTHROPIC_ANALYTICS_KEY }}
+          CORTEX_API_KEY: ${{ secrets.CORTEX_API_KEY }}
+        run: python scripts/sync-claude-spend.py

cortexapps_cli/solutions/ai-spend/README.md

@@ -0,0 +1,165 @@
+---
+name: AI Spend
+description: Track per-employee Claude AI spend in Cortex using custom metrics, with a full team hierarchy for rollup visibility.
+---
+
+# AI Spend
+
+Answers the question: **"How much are we spending on Claude AI, and who's spending it?"**
+
+Register every employee as a Cortex entity linked to their team, push weekly Claude spend as a custom metric, and roll costs up the org hierarchy — from individual → sub-team → top-level engineering.
+
+## How It Works
+
+```
+  ┌─────────────────────┐        every Monday 06:00 UTC
+  │   GitHub Actions    │◄──────────────────────────────────┐
+  │  sync-claude-spend  │                                   │
+  └────────┬────────────┘                             (cron schedule)
+           │
+           │ GET /v1/organizations/analytics/costs
+           ▼
+  ┌─────────────────────┐
+  │  Anthropic Claude   │   per-user spend for the week
+  │   Analytics API     │
+  └────────┬────────────┘
+           │
+           │ map email → employee-first-last
+           │ sum members → team rollups
+           ▼
+  ┌─────────────────────┐
+  │    Cortex API       │   POST ai-spend custom metric
+  │  Custom Metrics     │   per employee + per team
+  └────────┬────────────┘
+           │
+           ▼
+  ┌──────────────────────────────────────────────┐
+  │              Cortex Catalog                  │
+  │                                              │
+  │  team-engineering        $1,212/wk  Silver   │
+  │  ├── team-platform         $473/wk  Gold     │
+  │  │   ├── employee-alice      $291/wk         │
+  │  │   └── employee-bob        $182/wk         │
+  │  ├── team-frontend         $380/wk  Silver   │
+  │  │   ├── employee-carol      $245/wk         │
+  │  │   └── employee-david      $136/wk         │
+  │  └── team-data             $359/wk  Bronze   │
+  │      └── employee-emma      $359/wk          │
+  │                                              │
+  │  Scorecard: ai-spend-scorecard               │
+  │  Plugin:    team-ai-spend (per-team chart)   │
+  └──────────────────────────────────────────────┘
+```
+
+## What's Included
+
+| Resource | Tag / Key |
+|---|---|
+| Entity type | `employee` |
+| Relationship type | `team-member` (team → team\|employee) |
+| Teams | `team-engineering`, `team-platform`, `team-frontend`, `team-data` |
+| Employees | `employee-alice-chen`, `employee-bob-martinez`, `employee-carol-kim`, `employee-david-osei`, `employee-emma-johnson` |
+| Custom metric sample data | `ai-spend` (8 weeks, fictional, per-employee and team rollups) |
+| Plugin | `team-ai-spend` (team-scoped spend visualization) |
+| Scorecard | `ai-spend-scorecard` (bronze/silver/gold budget compliance) |
+| Sync script | `scripts/sync-claude-spend.py` |
+| GH Actions workflow | `.github/workflows/sync-claude-spend.yaml` |
+
+## Prerequisites
+
+Before installing, create the `ai-spend` custom metric definition in your Cortex instance:
+**Eng Intel → Custom Metrics → New Metric**, key: `ai-spend`.
+
+## Installation
+
+```
+cortex solutions install -s ai-spend
+```
+
+## After Installing
+
+**Create the team-member catalog**
+
+Enable the relationship type catalog so you can browse team membership from the Cortex UI:
+
+1. Go to **Settings → Entity Relationship Types → team-member**
+2. Click **Edit** and enable **Create relationship type catalog**
+3. Save
+
+**View the AI Spend Budget Compliance scorecard**
+
+An `ai-spend-scorecard` is installed automatically and tracks whether each team's weekly spend stays within budget:
+
+- **Bronze** — team has `ai-spend` metric data in the last 8 days and a budget set
+- **Silver** — spend is within 25% of budget
+- **Gold** — spend is at or under budget
+
+The sample data is pre-loaded with budgets that produce an interesting distribution: team-platform achieves Gold, team-frontend and team-engineering achieve Silver, and team-data achieves Bronze.
+
+To set a budget for a real team, add `ai-budget-weekly` as custom data on the team entity:
+
+```bash
+cortex custom-data add -t <team-tag> -k ai-budget-weekly -v <weekly-budget-dollars>
+```
+
+**View the Team AI Spend plugin**
+
+A `team-ai-spend` plugin is installed automatically and appears on every team entity page. It shows the team's total weekly AI spend and a per-member breakdown bar chart, pulling live data from the `ai-spend` custom metric.
+
+**Create a Tabular View for AI spend**
+
+Build a Data Explorer table to compare spend across employees and teams:
+
+1. Go to **Eng Intelligence → Data Explorer**
+2. Select the **Table** view
+3. Click **Add column**, find `ai-spend` under the **Custom** category, and click **View metric**
+4. Set **Group by → Team** and enable **Show hierarchy** to roll up spend to team level
+5. Click **Save As** to name and save the view for future use
+
+> Note: Tabular View creation is not yet available via API. It must be configured manually.
+
+**Set up live Claude spend sync**
+
+The sample entities include fictional spend data. To push real data from your Anthropic Claude Enterprise account weekly:
+
+1. **Get an Analytics API key:**
+   - Sign in to claude.ai as the **primary owner** of your organization
+   - Go to **Organization settings → API**
+   - Enable public API access and create an Analytics API key
+   - (Only the primary owner can create this key — admin role is not sufficient)
+
+2. **Add secrets to your GitHub repo:**
+   - `ANTHROPIC_ANALYTICS_KEY` — the Analytics API key from step 1
+   - `CORTEX_API_KEY` — your Cortex API key
+
+3. **Copy the workflow** to your repo's `.github/workflows/` directory:
+   ```bash
+   cp .github/workflows/sync-claude-spend.yaml <your-repo>/.github/workflows/
+   ```
+
+4. **Copy the script** to your repo's `scripts/` directory:
+   ```bash
+   cp scripts/sync-claude-spend.py <your-repo>/scripts/
+   ```
+
+The workflow runs every Monday at 06:00 UTC and can be triggered manually from the GitHub Actions tab.
+
+**Customize the email domain**
+
+The sync script maps `first.last@cortex.io` → `employee-first-last`. Set `EMAIL_DOMAIN` in the workflow env to match your company's domain:
+
+```yaml
+env:
+  EMAIL_DOMAIN: yourcompany.com
+```
+
+**Add your real employees**
+
+The sample entities are fictional. Add your real employees as catalog entities with `x-cortex-type: employee` and tag them `employee-<first>-<last>` to match the email mapping.
+
+**Notes**
+
+- Users who authenticate Claude Code with a personal API key (not Enterprise OAuth) show $0 spend in the Analytics API and are skipped automatically.
+- Cost data may take up to 24 hours to appear; dates at least 30 days old are considered final for billing purposes.
+- The `ai-spend` custom metric definition must currently be created manually before installing. A future release will support auto-creation of custom metric definitions as part of `cortex solutions install`.
+- The scorecard's Bronze rule uses a `P1Y` lookback to accommodate sample data. Once your weekly sync is running consistently, consider tightening it to `P8D` to ensure the rule only passes when data is fresh. The Silver and Gold rules use `P8D` and can similarly be adjusted to match your sync frequency.

cortexapps_cli/solutions/ai-spend/catalog/employee-alice-chen.yaml

@@ -0,0 +1,7 @@
+openapi: "3.0.0"
+info:
+  title: Alice Chen
+  x-cortex-tag: employee-alice-chen
+  x-cortex-type: employee
+  x-cortex-description: Platform Engineer
+  x-cortex-definition: {}

cortexapps_cli/solutions/ai-spend/catalog/employee-bob-martinez.yaml

@@ -0,0 +1,7 @@
+openapi: "3.0.0"
+info:
+  title: Bob Martinez
+  x-cortex-tag: employee-bob-martinez
+  x-cortex-type: employee
+  x-cortex-description: Platform Engineer
+  x-cortex-definition: {}

cortexapps_cli/solutions/ai-spend/catalog/employee-carol-kim.yaml

@@ -0,0 +1,7 @@
+openapi: "3.0.0"
+info:
+  title: Carol Kim
+  x-cortex-tag: employee-carol-kim
+  x-cortex-type: employee
+  x-cortex-description: Frontend Engineer
+  x-cortex-definition: {}

cortexapps_cli/solutions/ai-spend/catalog/employee-david-osei.yaml

@@ -0,0 +1,7 @@
+openapi: "3.0.0"
+info:
+  title: David Osei
+  x-cortex-tag: employee-david-osei
+  x-cortex-type: employee
+  x-cortex-description: Frontend Engineer
+  x-cortex-definition: {}

cortexapps_cli/solutions/ai-spend/catalog/employee-emma-johnson.yaml

@@ -0,0 +1,7 @@
+openapi: "3.0.0"
+info:
+  title: Emma Johnson
+  x-cortex-tag: employee-emma-johnson
+  x-cortex-type: employee
+  x-cortex-description: Data Engineer
+  x-cortex-definition: {}