diff --git a/cortexapps_cli/commands/backup.py b/cortexapps_cli/commands/backup.py index a0fc377..a36d0f4 100644 --- a/cortexapps_cli/commands/backup.py +++ b/cortexapps_cli/commands/backup.py @@ -471,6 +471,47 @@ def import_relationships_file(file_info): return ("entity-relationships", len(results) - failed_count, [(fp, et, em) for rt, fp, et, em in results if et]) +def _import_custom_metrics(ctx, directory): + imported = 0 + failed = [] + if os.path.isdir(directory): + print("Processing: " + directory) + client = ctx.obj["client"] + for filename in sorted(os.listdir(directory)): + if not filename.endswith(".json"): + continue + file_path = os.path.join(directory, filename) + if not os.path.isfile(file_path): + continue + metric_key = filename[:-5] # strip .json + try: + print(" Importing: " + filename) + with open(file_path) as f: + data = json.load(f) + + # Group flat values list by entityTag + grouped = {} + for entry in data.get("values", []): + tag = entry["entityTag"] + if tag not in grouped: + grouped[tag] = [] + grouped[tag].append({ + "timestamp": entry["timestamp"], + "value": entry["value"], + }) + + # Call per-entity bulk endpoint once per entity + for entity_tag, series in grouped.items(): + client.post( + f"api/v1/eng-intel/custom-metrics/{metric_key}/entity/{entity_tag}/bulk", + data={"series": series}, + ) + imported += 1 + except Exception as e: + print(f" Failed to import {filename}: {type(e).__name__} - {str(e)}") + failed.append((file_path, type(e).__name__, str(e))) + return ("custom-metrics", imported, failed) + def _has_relationships(file_path): """Check if a catalog YAML file contains x-cortex-relationships.""" try: @@ -760,6 +801,7 @@ def import_tenant( all_stats.append(_import_entity_relationship_types(ctx, directory + "/entity-relationship-types")) all_stats.append(_import_catalog(ctx, directory + "/catalog")) all_stats.append(_import_entity_relationships(ctx, directory + "/entity-relationships")) + all_stats.append(_import_custom_metrics(ctx, directory + "/custom-metrics")) all_stats.append(_import_plugins(ctx, directory + "/plugins")) all_stats.append(_import_scorecards(ctx, directory + "/scorecards")) all_stats.append(_import_workflows(ctx, directory + "/workflows")) @@ -811,6 +853,8 @@ def import_tenant( elif import_type == "entity-relationships": # These need special handling - would need the relationship type print(f"# Manual retry needed for entity-relationships: {file_path}") + elif import_type == "custom-metrics": + print(f"# Manual retry needed for custom-metrics: {file_path}") elif import_type == "plugins": print(f"cortex plugins create --force -f \"{file_path}\"") elif import_type == "scorecards": diff --git a/cortexapps_cli/commands/plugins.py b/cortexapps_cli/commands/plugins.py index 65d8343..f31cf7e 100644 --- a/cortexapps_cli/commands/plugins.py +++ b/cortexapps_cli/commands/plugins.py @@ -87,6 +87,8 @@ def create( # Remove the 'tag' attribute if it exists data.pop("tag", None) r = client.put("api/v1/plugins/" + tag, data, raw_response=True) + else: + r = client.post("api/v1/plugins", data, raw_response=True) else: r = client.post("api/v1/plugins", data, raw_response=True) @@ -142,12 +144,18 @@ def get( def replace( ctx: typer.Context, file_input: Annotated[typer.FileText, typer.Option("--file", "-f", help="File containing contents of plugin using schema defined at https://docs.cortex.io/docs/api/create-plugin")] = None, - tag_or_id: str = typer.Option(..., "--tag-or-id", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity.") + tag_or_id: str = typer.Option(None, "--tag-or-id", "-t", help="The tag or ID of the plugin to replace. Defaults to the tag field in the file."), ): """ Replace an existing plugin by tag """ client = ctx.obj["client"] - - client.put("api/v1/plugins/"+ tag_or_id, data=file_input.read()) + + data = json.loads(file_input.read()) + resolved = tag_or_id or data.get("tag") + if not resolved: + typer.echo("Error: --tag-or-id is required when the file does not contain a 'tag' field.") + raise typer.Exit(1) + + client.put("api/v1/plugins/" + resolved, data=data) diff --git a/cortexapps_cli/commands/solutions.py b/cortexapps_cli/commands/solutions.py index 32ef4a3..c8e7d1f 100644 --- a/cortexapps_cli/commands/solutions.py +++ b/cortexapps_cli/commands/solutions.py @@ -280,6 +280,7 @@ def _collect_solution_resources(path: Path) -> dict[str, list[str]]: "catalog": [], "scorecards": [], "workflows": [], + "plugins": [], } for kind in resources: subdir = path / kind @@ -336,7 +337,7 @@ def _run_uninstall(client, path: Path, yes: bool) -> None: return typer.echo("\nThis will remove the following resources:") - for kind in ("workflows", "scorecards", "catalog", "entity-relationship-types", "entity-types"): + for kind in ("workflows", "scorecards", "plugins", "catalog", "entity-relationship-types", "entity-types"): count = len(resources[kind]) if count: typer.echo(f" {kind}: {count}") @@ -352,6 +353,7 @@ def _run_uninstall(client, path: Path, yes: bool) -> None: steps = [ ("workflows", lambda t: f"api/v1/workflows/{t}"), ("scorecards", lambda t: f"api/v1/scorecards/{t}"), + ("plugins", lambda t: f"api/v1/plugins/{t}"), ("catalog", lambda t: f"api/v1/catalog/{t}"), ("entity-relationship-types", lambda t: f"api/v1/relationship-types/{t}"), ("entity-types", lambda t: f"api/v1/catalog/definitions/{t}"), diff --git a/cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml b/cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml new file mode 100644 index 0000000..7f5df8f --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml @@ -0,0 +1,27 @@ +name: Sync Claude AI Spend to Cortex + +on: + schedule: + - cron: "0 6 * * 1" # Every Monday at 06:00 UTC + workflow_dispatch: # Allow manual runs from the Actions tab + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install requests + + - name: Sync Claude spend to Cortex + env: + ANTHROPIC_ANALYTICS_KEY: ${{ secrets.ANTHROPIC_ANALYTICS_KEY }} + CORTEX_API_KEY: ${{ secrets.CORTEX_API_KEY }} + run: python scripts/sync-claude-spend.py diff --git a/cortexapps_cli/solutions/ai-spend/README.md b/cortexapps_cli/solutions/ai-spend/README.md new file mode 100644 index 0000000..cd5fc16 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/README.md @@ -0,0 +1,165 @@ +--- +name: AI Spend +description: Track per-employee Claude AI spend in Cortex using custom metrics, with a full team hierarchy for rollup visibility. +--- + +# AI Spend + +Answers the question: **"How much are we spending on Claude AI, and who's spending it?"** + +Register every employee as a Cortex entity linked to their team, push weekly Claude spend as a custom metric, and roll costs up the org hierarchy — from individual → sub-team → top-level engineering. + +## How It Works + +``` + ┌─────────────────────┐ every Monday 06:00 UTC + │ GitHub Actions │◄──────────────────────────────────┐ + │ sync-claude-spend │ │ + └────────┬────────────┘ (cron schedule) + │ + │ GET /v1/organizations/analytics/costs + ▼ + ┌─────────────────────┐ + │ Anthropic Claude │ per-user spend for the week + │ Analytics API │ + └────────┬────────────┘ + │ + │ map email → employee-first-last + │ sum members → team rollups + ▼ + ┌─────────────────────┐ + │ Cortex API │ POST ai-spend custom metric + │ Custom Metrics │ per employee + per team + └────────┬────────────┘ + │ + ▼ + ┌──────────────────────────────────────────────┐ + │ Cortex Catalog │ + │ │ + │ team-engineering $1,212/wk Silver │ + │ ├── team-platform $473/wk Gold │ + │ │ ├── employee-alice $291/wk │ + │ │ └── employee-bob $182/wk │ + │ ├── team-frontend $380/wk Silver │ + │ │ ├── employee-carol $245/wk │ + │ │ └── employee-david $136/wk │ + │ └── team-data $359/wk Bronze │ + │ └── employee-emma $359/wk │ + │ │ + │ Scorecard: ai-spend-scorecard │ + │ Plugin: team-ai-spend (per-team chart) │ + └──────────────────────────────────────────────┘ +``` + +## What's Included + +| Resource | Tag / Key | +|---|---| +| Entity type | `employee` | +| Relationship type | `team-member` (team → team\|employee) | +| Teams | `team-engineering`, `team-platform`, `team-frontend`, `team-data` | +| Employees | `employee-alice-chen`, `employee-bob-martinez`, `employee-carol-kim`, `employee-david-osei`, `employee-emma-johnson` | +| Custom metric sample data | `ai-spend` (8 weeks, fictional, per-employee and team rollups) | +| Plugin | `team-ai-spend` (team-scoped spend visualization) | +| Scorecard | `ai-spend-scorecard` (bronze/silver/gold budget compliance) | +| Sync script | `scripts/sync-claude-spend.py` | +| GH Actions workflow | `.github/workflows/sync-claude-spend.yaml` | + +## Prerequisites + +Before installing, create the `ai-spend` custom metric definition in your Cortex instance: +**Eng Intel → Custom Metrics → New Metric**, key: `ai-spend`. + +## Installation + +``` +cortex solutions install -s ai-spend +``` + +## After Installing + +**Create the team-member catalog** + +Enable the relationship type catalog so you can browse team membership from the Cortex UI: + +1. Go to **Settings → Entity Relationship Types → team-member** +2. Click **Edit** and enable **Create relationship type catalog** +3. Save + +**View the AI Spend Budget Compliance scorecard** + +An `ai-spend-scorecard` is installed automatically and tracks whether each team's weekly spend stays within budget: + +- **Bronze** — team has `ai-spend` metric data in the last 8 days and a budget set +- **Silver** — spend is within 25% of budget +- **Gold** — spend is at or under budget + +The sample data is pre-loaded with budgets that produce an interesting distribution: team-platform achieves Gold, team-frontend and team-engineering achieve Silver, and team-data achieves Bronze. + +To set a budget for a real team, add `ai-budget-weekly` as custom data on the team entity: + +```bash +cortex custom-data add -t -k ai-budget-weekly -v +``` + +**View the Team AI Spend plugin** + +A `team-ai-spend` plugin is installed automatically and appears on every team entity page. It shows the team's total weekly AI spend and a per-member breakdown bar chart, pulling live data from the `ai-spend` custom metric. + +**Create a Tabular View for AI spend** + +Build a Data Explorer table to compare spend across employees and teams: + +1. Go to **Eng Intelligence → Data Explorer** +2. Select the **Table** view +3. Click **Add column**, find `ai-spend` under the **Custom** category, and click **View metric** +4. Set **Group by → Team** and enable **Show hierarchy** to roll up spend to team level +5. Click **Save As** to name and save the view for future use + +> Note: Tabular View creation is not yet available via API. It must be configured manually. + +**Set up live Claude spend sync** + +The sample entities include fictional spend data. To push real data from your Anthropic Claude Enterprise account weekly: + +1. **Get an Analytics API key:** + - Sign in to claude.ai as the **primary owner** of your organization + - Go to **Organization settings → API** + - Enable public API access and create an Analytics API key + - (Only the primary owner can create this key — admin role is not sufficient) + +2. **Add secrets to your GitHub repo:** + - `ANTHROPIC_ANALYTICS_KEY` — the Analytics API key from step 1 + - `CORTEX_API_KEY` — your Cortex API key + +3. **Copy the workflow** to your repo's `.github/workflows/` directory: + ```bash + cp .github/workflows/sync-claude-spend.yaml /.github/workflows/ + ``` + +4. **Copy the script** to your repo's `scripts/` directory: + ```bash + cp scripts/sync-claude-spend.py /scripts/ + ``` + +The workflow runs every Monday at 06:00 UTC and can be triggered manually from the GitHub Actions tab. + +**Customize the email domain** + +The sync script maps `first.last@cortex.io` → `employee-first-last`. Set `EMAIL_DOMAIN` in the workflow env to match your company's domain: + +```yaml +env: + EMAIL_DOMAIN: yourcompany.com +``` + +**Add your real employees** + +The sample entities are fictional. Add your real employees as catalog entities with `x-cortex-type: employee` and tag them `employee--` to match the email mapping. + +**Notes** + +- Users who authenticate Claude Code with a personal API key (not Enterprise OAuth) show $0 spend in the Analytics API and are skipped automatically. +- Cost data may take up to 24 hours to appear; dates at least 30 days old are considered final for billing purposes. +- The `ai-spend` custom metric definition must currently be created manually before installing. A future release will support auto-creation of custom metric definitions as part of `cortex solutions install`. +- The scorecard's Bronze rule uses a `P1Y` lookback to accommodate sample data. Once your weekly sync is running consistently, consider tightening it to `P8D` to ensure the rule only passes when data is fresh. The Silver and Gold rules use `P8D` and can similarly be adjusted to match your sync frequency. diff --git a/cortexapps_cli/solutions/ai-spend/catalog/employee-alice-chen.yaml b/cortexapps_cli/solutions/ai-spend/catalog/employee-alice-chen.yaml new file mode 100644 index 0000000..9aea3ce --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/employee-alice-chen.yaml @@ -0,0 +1,7 @@ +openapi: "3.0.0" +info: + title: Alice Chen + x-cortex-tag: employee-alice-chen + x-cortex-type: employee + x-cortex-description: Platform Engineer + x-cortex-definition: {} diff --git a/cortexapps_cli/solutions/ai-spend/catalog/employee-bob-martinez.yaml b/cortexapps_cli/solutions/ai-spend/catalog/employee-bob-martinez.yaml new file mode 100644 index 0000000..c14f526 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/employee-bob-martinez.yaml @@ -0,0 +1,7 @@ +openapi: "3.0.0" +info: + title: Bob Martinez + x-cortex-tag: employee-bob-martinez + x-cortex-type: employee + x-cortex-description: Platform Engineer + x-cortex-definition: {} diff --git a/cortexapps_cli/solutions/ai-spend/catalog/employee-carol-kim.yaml b/cortexapps_cli/solutions/ai-spend/catalog/employee-carol-kim.yaml new file mode 100644 index 0000000..9ba9a31 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/employee-carol-kim.yaml @@ -0,0 +1,7 @@ +openapi: "3.0.0" +info: + title: Carol Kim + x-cortex-tag: employee-carol-kim + x-cortex-type: employee + x-cortex-description: Frontend Engineer + x-cortex-definition: {} diff --git a/cortexapps_cli/solutions/ai-spend/catalog/employee-david-osei.yaml b/cortexapps_cli/solutions/ai-spend/catalog/employee-david-osei.yaml new file mode 100644 index 0000000..497e061 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/employee-david-osei.yaml @@ -0,0 +1,7 @@ +openapi: "3.0.0" +info: + title: David Osei + x-cortex-tag: employee-david-osei + x-cortex-type: employee + x-cortex-description: Frontend Engineer + x-cortex-definition: {} diff --git a/cortexapps_cli/solutions/ai-spend/catalog/employee-emma-johnson.yaml b/cortexapps_cli/solutions/ai-spend/catalog/employee-emma-johnson.yaml new file mode 100644 index 0000000..4123088 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/employee-emma-johnson.yaml @@ -0,0 +1,7 @@ +openapi: "3.0.0" +info: + title: Emma Johnson + x-cortex-tag: employee-emma-johnson + x-cortex-type: employee + x-cortex-description: Data Engineer + x-cortex-definition: {} diff --git a/cortexapps_cli/solutions/ai-spend/catalog/team-data.yaml b/cortexapps_cli/solutions/ai-spend/catalog/team-data.yaml new file mode 100644 index 0000000..a38f0d9 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/team-data.yaml @@ -0,0 +1,20 @@ +openapi: "3.0.0" +info: + title: Data + x-cortex-tag: team-data + x-cortex-type: team + x-cortex-description: Data engineering team + x-cortex-definition: {} + x-cortex-groups: + - ai-spend-demo + x-cortex-team: + members: + - name: Emma Johnson + email: emma.johnson@cortex.io + x-cortex-custom-data: + - key: ai-budget-weekly + value: 280 + x-cortex-relationships: + - type: team-member + destinations: + - tag: employee-emma-johnson diff --git a/cortexapps_cli/solutions/ai-spend/catalog/team-engineering.yaml b/cortexapps_cli/solutions/ai-spend/catalog/team-engineering.yaml new file mode 100644 index 0000000..8ec54e3 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/team-engineering.yaml @@ -0,0 +1,30 @@ +openapi: "3.0.0" +info: + title: Engineering + x-cortex-tag: team-engineering + x-cortex-type: team + x-cortex-description: Top-level engineering organization + x-cortex-definition: {} + x-cortex-groups: + - ai-spend-demo + x-cortex-team: + members: + - name: Alice Chen + email: alice.chen@cortex.io + - name: Bob Martinez + email: bob.martinez@cortex.io + - name: Carol Kim + email: carol.kim@cortex.io + - name: David Osei + email: david.osei@cortex.io + - name: Emma Johnson + email: emma.johnson@cortex.io + x-cortex-custom-data: + - key: ai-budget-weekly + value: 1100 + x-cortex-relationships: + - type: team-member + destinations: + - tag: team-platform + - tag: team-frontend + - tag: team-data diff --git a/cortexapps_cli/solutions/ai-spend/catalog/team-frontend.yaml b/cortexapps_cli/solutions/ai-spend/catalog/team-frontend.yaml new file mode 100644 index 0000000..ac3839e --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/team-frontend.yaml @@ -0,0 +1,23 @@ +openapi: "3.0.0" +info: + title: Frontend + x-cortex-tag: team-frontend + x-cortex-type: team + x-cortex-description: Frontend engineering team + x-cortex-definition: {} + x-cortex-groups: + - ai-spend-demo + x-cortex-team: + members: + - name: Carol Kim + email: carol.kim@cortex.io + - name: David Osei + email: david.osei@cortex.io + x-cortex-custom-data: + - key: ai-budget-weekly + value: 360 + x-cortex-relationships: + - type: team-member + destinations: + - tag: employee-carol-kim + - tag: employee-david-osei diff --git a/cortexapps_cli/solutions/ai-spend/catalog/team-platform.yaml b/cortexapps_cli/solutions/ai-spend/catalog/team-platform.yaml new file mode 100644 index 0000000..1f83989 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/catalog/team-platform.yaml @@ -0,0 +1,23 @@ +openapi: "3.0.0" +info: + title: Platform + x-cortex-tag: team-platform + x-cortex-type: team + x-cortex-description: Platform engineering team + x-cortex-definition: {} + x-cortex-groups: + - ai-spend-demo + x-cortex-team: + members: + - name: Alice Chen + email: alice.chen@cortex.io + - name: Bob Martinez + email: bob.martinez@cortex.io + x-cortex-custom-data: + - key: ai-budget-weekly + value: 480 + x-cortex-relationships: + - type: team-member + destinations: + - tag: employee-alice-chen + - tag: employee-bob-martinez diff --git a/cortexapps_cli/solutions/ai-spend/custom-metrics/ai-spend.json b/cortexapps_cli/solutions/ai-spend/custom-metrics/ai-spend.json new file mode 100644 index 0000000..ec429f2 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/custom-metrics/ai-spend.json @@ -0,0 +1,84 @@ +{ + "values": [ + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-09T00:00:00", "value": 38.20 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-16T00:00:00", "value": 56.40 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-23T00:00:00", "value": 79.80 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-30T00:00:00", "value": 127.30 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-07T00:00:00", "value": 179.60 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-14T00:00:00", "value": 234.80 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-21T00:00:00", "value": 268.90 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-28T00:00:00", "value": 291.40 }, + + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-09T00:00:00", "value": 24.10 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-16T00:00:00", "value": 31.80 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-23T00:00:00", "value": 54.20 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-30T00:00:00", "value": 75.40 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-07T00:00:00", "value": 118.30 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-14T00:00:00", "value": 141.70 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-21T00:00:00", "value": 173.90 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-28T00:00:00", "value": 181.60 }, + + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-09T00:00:00", "value": 27.30 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-16T00:00:00", "value": 47.10 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-23T00:00:00", "value": 66.80 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-30T00:00:00", "value": 108.40 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-07T00:00:00", "value": 157.20 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-14T00:00:00", "value": 189.60 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-21T00:00:00", "value": 231.50 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-28T00:00:00", "value": 244.70 }, + + { "entityTag": "employee-david-osei", "timestamp": "2026-06-09T00:00:00", "value": 14.80 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-06-16T00:00:00", "value": 27.30 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-06-23T00:00:00", "value": 41.20 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-06-30T00:00:00", "value": 55.60 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-07T00:00:00", "value": 89.40 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-14T00:00:00", "value": 104.80 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-21T00:00:00", "value": 129.70 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-28T00:00:00", "value": 135.50 }, + + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-09T00:00:00", "value": 46.20 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-16T00:00:00", "value": 62.90 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-23T00:00:00", "value": 107.40 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-30T00:00:00", "value": 149.80 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-07T00:00:00", "value": 231.60 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-14T00:00:00", "value": 279.30 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-21T00:00:00", "value": 341.20 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-28T00:00:00", "value": 358.70 }, + + { "entityTag": "team-platform", "timestamp": "2026-06-09T00:00:00", "value": 62.30 }, + { "entityTag": "team-platform", "timestamp": "2026-06-16T00:00:00", "value": 88.20 }, + { "entityTag": "team-platform", "timestamp": "2026-06-23T00:00:00", "value": 134.00 }, + { "entityTag": "team-platform", "timestamp": "2026-06-30T00:00:00", "value": 202.70 }, + { "entityTag": "team-platform", "timestamp": "2026-07-07T00:00:00", "value": 297.90 }, + { "entityTag": "team-platform", "timestamp": "2026-07-14T00:00:00", "value": 376.50 }, + { "entityTag": "team-platform", "timestamp": "2026-07-21T00:00:00", "value": 442.80 }, + { "entityTag": "team-platform", "timestamp": "2026-07-28T00:00:00", "value": 473.00 }, + + { "entityTag": "team-frontend", "timestamp": "2026-06-09T00:00:00", "value": 42.10 }, + { "entityTag": "team-frontend", "timestamp": "2026-06-16T00:00:00", "value": 74.40 }, + { "entityTag": "team-frontend", "timestamp": "2026-06-23T00:00:00", "value": 108.00 }, + { "entityTag": "team-frontend", "timestamp": "2026-06-30T00:00:00", "value": 164.00 }, + { "entityTag": "team-frontend", "timestamp": "2026-07-07T00:00:00", "value": 246.60 }, + { "entityTag": "team-frontend", "timestamp": "2026-07-14T00:00:00", "value": 294.40 }, + { "entityTag": "team-frontend", "timestamp": "2026-07-21T00:00:00", "value": 361.20 }, + { "entityTag": "team-frontend", "timestamp": "2026-07-28T00:00:00", "value": 380.20 }, + + { "entityTag": "team-data", "timestamp": "2026-06-09T00:00:00", "value": 46.20 }, + { "entityTag": "team-data", "timestamp": "2026-06-16T00:00:00", "value": 62.90 }, + { "entityTag": "team-data", "timestamp": "2026-06-23T00:00:00", "value": 107.40 }, + { "entityTag": "team-data", "timestamp": "2026-06-30T00:00:00", "value": 149.80 }, + { "entityTag": "team-data", "timestamp": "2026-07-07T00:00:00", "value": 231.60 }, + { "entityTag": "team-data", "timestamp": "2026-07-14T00:00:00", "value": 279.30 }, + { "entityTag": "team-data", "timestamp": "2026-07-21T00:00:00", "value": 341.20 }, + { "entityTag": "team-data", "timestamp": "2026-07-28T00:00:00", "value": 358.70 }, + + { "entityTag": "team-engineering", "timestamp": "2026-06-09T00:00:00", "value": 150.60 }, + { "entityTag": "team-engineering", "timestamp": "2026-06-16T00:00:00", "value": 225.50 }, + { "entityTag": "team-engineering", "timestamp": "2026-06-23T00:00:00", "value": 349.40 }, + { "entityTag": "team-engineering", "timestamp": "2026-06-30T00:00:00", "value": 516.50 }, + { "entityTag": "team-engineering", "timestamp": "2026-07-07T00:00:00", "value": 776.10 }, + { "entityTag": "team-engineering", "timestamp": "2026-07-14T00:00:00", "value": 950.20 }, + { "entityTag": "team-engineering", "timestamp": "2026-07-21T00:00:00", "value": 1145.20 }, + { "entityTag": "team-engineering", "timestamp": "2026-07-28T00:00:00", "value": 1211.90 } + ] +} diff --git a/cortexapps_cli/solutions/ai-spend/entity-relationship-types/team-member.json b/cortexapps_cli/solutions/ai-spend/entity-relationship-types/team-member.json new file mode 100644 index 0000000..05b12c6 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/entity-relationship-types/team-member.json @@ -0,0 +1,20 @@ +{ + "tag": "team-member", + "name": "Team Member", + "description": "Links a team to its direct members, which can be sub-teams or individual employees. Use this single relationship type to walk the full org hierarchy in the catalog.", + "definitionLocation": "SOURCE", + "isSingleSource": false, + "isSingleDestination": false, + "allowCycles": false, + "sourcesFilter": { + "include": true, + "types": ["team"], + "providers": [] + }, + "destinationsFilter": { + "include": true, + "types": ["team", "employee"], + "providers": [] + }, + "inheritances": [] +} diff --git a/cortexapps_cli/solutions/ai-spend/entity-types/employee.json b/cortexapps_cli/solutions/ai-spend/entity-types/employee.json new file mode 100644 index 0000000..e1107d7 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/entity-types/employee.json @@ -0,0 +1,7 @@ +{ + "type": "employee", + "name": "Employee", + "description": "A member of the organization. Used to track AI tool usage and spend per person.", + "iconTag": "Cortex-builtin::Person", + "schema": {"type": "object", "properties": {}} +} diff --git a/cortexapps_cli/solutions/ai-spend/plugins/team-ai-spend.json b/cortexapps_cli/solutions/ai-spend/plugins/team-ai-spend.json new file mode 100644 index 0000000..1bad188 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/plugins/team-ai-spend.json @@ -0,0 +1,14 @@ +{ + "tag": "team-ai-spend", + "name": "Team AI Spend", + "description": "Visualizes per-member Claude AI spend for the team with a weekly total and per-member breakdown chart.", + "isDraft": false, + "minimumRoleRequired": "VIEWER", + "contexts": [ + { "type": "ENTITY", "entityFilter": { "type": "CQL_FILTER", "category": "Team", "query": "hasGroup(\"ai-spend-demo\")" } } + ], + "proxyTag": null, + "iconTag": null, + "version": null, + "blob": "\n\n\n \n \n Team AI Spend\n \n \n \n\n\n
Loading AI spend data\u2026
\n
\n
\n
\n
Team Weekly AI Spend
\n
$0
\n
\n
\n
Per-Member Breakdown
\n \n
\n\n \n\n" +} diff --git a/cortexapps_cli/solutions/ai-spend/scorecards/ai-spend-scorecard.yaml b/cortexapps_cli/solutions/ai-spend/scorecards/ai-spend-scorecard.yaml new file mode 100644 index 0000000..ae45c38 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/scorecards/ai-spend-scorecard.yaml @@ -0,0 +1,58 @@ +tag: ai-spend-scorecard +name: AI Spend Budget Compliance +description: Tracks whether team AI spend stays within the team's weekly budget, with bronze for data tracking, silver for near-budget compliance, and gold for on-budget status. +draft: false +notifications: + enabled: true + scoreDropNotificationsEnabled: true +exemptions: + enabled: true + autoApprove: false + userSpecificNotifications: false +evaluation: + window: 24 +ladder: + name: Default Ladder + levels: + - name: Bronze + rank: 1 + description: Team is tracking AI spend with the ai-spend custom metric and has a weekly budget set. + color: "#CD7F32" + - name: Silver + rank: 2 + description: Team AI spend is within 25% of the weekly budget. + color: "#C0C0C0" + - name: Gold + rank: 3 + description: Team AI spend is at or under the weekly budget. + color: "#D7AC58" +filter: + kind: GENERIC + types: + include: + - team + query: "hasGroup(\"ai-spend-demo\")" +rules: + - title: AI spend data is being tracked + description: The team has ai-spend custom metric data within the last year, indicating the sync script has run at least once. Adjust the P1Y lookback to a tighter window (e.g. P8D) once the weekly sync is running consistently. + expression: customMetrics(key="ai-spend", lookback = duration("P1Y")).length > 0 + weight: 1 + level: Bronze + + - title: Weekly budget is defined + description: The team has a weekly AI budget set via the ai-budget-weekly custom data key. Without a budget, compliance cannot be measured. + expression: custom("ai-budget-weekly") != null + weight: 1 + level: Bronze + + - title: Spend is within 25% of budget + description: The team's weekly AI spend is no more than 25% over the budget. The P8D lookback averages the last week of data — adjust to match your sync frequency if needed. + expression: customMetrics(key="ai-spend", lookback=duration("P8D")).map((m) => m.value).average() <= jq(custom("ai-budget-weekly"), ". | tonumber") * 1.25 + weight: 1 + level: Silver + + - title: Spend is at or under budget + description: The team's weekly AI spend is at or under the budget. This is the target state for all teams. The P8D lookback averages the last week of data — adjust to match your sync frequency if needed. + expression: customMetrics(key="ai-spend", lookback=duration("P8D")).map((m) => m.value).average() <= jq(custom("ai-budget-weekly"), ". | tonumber") + weight: 1 + level: Gold diff --git a/cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py b/cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py new file mode 100644 index 0000000..28aa493 --- /dev/null +++ b/cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +""" +sync-claude-spend.py + +Pulls per-user spend from the Anthropic Claude Enterprise Analytics API +and pushes weekly cost data to Cortex as custom metric data points. +Also computes per-team rollups by walking the team-member relationship +hierarchy and writing the aggregated spend to the ai-spend custom metric +on each team entity. + +Requirements: + pip install requests + +Environment variables: + ANTHROPIC_ANALYTICS_KEY Required. Analytics API key from claude.ai org settings. + Only the primary owner can create this key at: + claude.ai > Organization settings > API + CORTEX_API_KEY Required. Cortex API key. + CORTEX_BASE_URL Optional. Defaults to https://api.getcortexapp.com + EMAIL_DOMAIN Optional. Domain to strip from emails. Defaults to cortex.io + +Usage: + python sync-claude-spend.py + python sync-claude-spend.py --start 2026-07-21 --end 2026-07-28 + +Notes: + - Users who authenticate via API key (not Enterprise OAuth) will show $0 spend + in the Analytics API and are skipped automatically. + - The Cortex custom metric definition for "ai-spend" must already exist in your + Cortex instance before running this script. Create it in the Cortex UI under + Eng Intel > Custom Metrics. + - Team rollups require team entities to have team-member relationships pointing + to employee entities (or other teams, which are resolved recursively). +""" + +import argparse +import os +import sys +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +import requests + +ANTHROPIC_BASE_URL = "https://api.anthropic.com" +ANTHROPIC_VERSION = "2023-06-01" +CORTEX_METRIC_KEY = "ai-spend" +TEAM_MEMBER_RELATIONSHIP = "team-member" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Sync Claude Enterprise spend to Cortex custom metrics" + ) + parser.add_argument( + "--start", + help="Start date YYYY-MM-DD (default: 7 days ago)", + default=None, + ) + parser.add_argument( + "--end", + help="End date YYYY-MM-DD (default: yesterday)", + default=None, + ) + return parser.parse_args() + + +def get_env(key, required=True, default=None): + value = os.environ.get(key, default) + if required and not value: + print(f"ERROR: Environment variable {key} is required", file=sys.stderr) + sys.exit(1) + return value + + +def email_to_entity_tag(email, domain): + """ + Maps first.last@domain -> employee-first-last. + Returns None if email doesn't match the expected domain or format. + """ + if not email.endswith(f"@{domain}"): + return None + local = email.split("@")[0] + parts = local.split(".") + if len(parts) != 2: + return None + return f"employee-{parts[0]}-{parts[1]}" + + +def fetch_claude_spend(analytics_key, start_date, end_date): + """ + Fetch per-user cost data from the Claude Enterprise Analytics API. + + Returns list of dicts: {"email": str, "cost_dollars": float} + Only includes records where cost > 0. + + Endpoint: GET /v1/organizations/analytics/costs + Verify exact query parameters once an Analytics API key is available. + """ + headers = { + "x-api-key": analytics_key, + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json", + } + + results = [] + cursor = None + + while True: + params = { + "starting_at": start_date, + "ending_at": end_date, + } + if cursor: + params["page"] = cursor + + url = f"{ANTHROPIC_BASE_URL}/v1/organizations/analytics/costs" + response = requests.get(url, headers=headers, params=params, timeout=30) + response.raise_for_status() + body = response.json() + + for record in body.get("data", []): + actor = record.get("actor", {}) + email = actor.get("email_address") + if not email: + continue + + # Cost is returned as a decimal string in cents (e.g. "14250.000000" = $142.50) + cost_str = record.get("cost", "0") + try: + cost_dollars = float(cost_str) / 100 + except (ValueError, TypeError): + cost_dollars = 0.0 + + if cost_dollars > 0: + results.append({"email": email, "cost_dollars": cost_dollars}) + + if not body.get("has_more"): + break + cursor = body.get("next_page") + + return results + + +def fetch_team_member_relationships(cortex_api_key, cortex_base_url): + """ + Fetch all team-member relationship instances from Cortex. + + Returns a dict mapping source_tag -> list of dicts {"tag": str, "type": str} + where type is the entity type (e.g. "employee", "team"). + """ + url = f"{cortex_base_url}/api/v1/relationships/{TEAM_MEMBER_RELATIONSHIP}" + headers = { + "Authorization": f"Bearer {cortex_api_key}", + "Content-Type": "application/json", + } + adjacency = defaultdict(list) + page = 0 + while True: + params = {"pageSize": 200, "page": page} + response = requests.get(url, headers=headers, params=params, timeout=30) + response.raise_for_status() + body = response.json() + for rel in body.get("relationships", []): + src = rel.get("sourceEntity", {}) + dst = rel.get("destinationEntity", {}) + if src.get("tag") and dst.get("tag"): + adjacency[src["tag"]].append({ + "tag": dst["tag"], + "type": dst.get("type", ""), + }) + if not body.get("hasNextPage") and not body.get("has_more"): + break + page += 1 + return adjacency + + +def collect_leaf_employees(team_tag, adjacency, visited=None): + """ + DFS walk of the team-member hierarchy to collect all leaf employee tags. + + Teams that point to sub-teams are resolved recursively; cycles are guarded + with the visited set. Returns a set of employee entity tags. + """ + if visited is None: + visited = set() + if team_tag in visited: + return set() + visited.add(team_tag) + + employees = set() + for member in adjacency.get(team_tag, []): + if member["type"] == "employee": + employees.add(member["tag"]) + else: + # Recurse into sub-teams + employees |= collect_leaf_employees(member["tag"], adjacency, visited) + return employees + + +def push_to_cortex(cortex_api_key, cortex_base_url, entity_tag, series): + """ + Push spend data points for a single entity to Cortex. + + series: list of {"timestamp": str, "value": float} + Calls: POST /api/v1/eng-intel/custom-metrics/{key}/entity/{tag}/bulk + """ + url = ( + f"{cortex_base_url}/api/v1/eng-intel/custom-metrics" + f"/{CORTEX_METRIC_KEY}/entity/{entity_tag}/bulk" + ) + headers = { + "Authorization": f"Bearer {cortex_api_key}", + "Content-Type": "application/json", + } + response = requests.post( + url, headers=headers, json={"series": series}, timeout=30 + ) + response.raise_for_status() + + +def main(): + args = parse_args() + + analytics_key = get_env("ANTHROPIC_ANALYTICS_KEY") + cortex_api_key = get_env("CORTEX_API_KEY") + cortex_base_url = get_env( + "CORTEX_BASE_URL", required=False, default="https://api.getcortexapp.com" + ) + email_domain = get_env("EMAIL_DOMAIN", required=False, default="cortex.io") + + today = datetime.now(timezone.utc).date() + start_date = args.start or str(today - timedelta(days=7)) + end_date = args.end or str(today - timedelta(days=1)) + # Use end_date as the metric timestamp (represents the week ending on this date) + timestamp = f"{end_date}T00:00:00" + + print(f"Fetching Claude spend from {start_date} to {end_date}...") + + try: + spend_records = fetch_claude_spend(analytics_key, start_date, end_date) + except requests.HTTPError as e: + print(f"ERROR: Failed to fetch spend data from Anthropic: {e}", file=sys.stderr) + sys.exit(1) + + # Map emails to entity tags; collect skips + entity_series = defaultdict(list) + skipped = [] + + for record in spend_records: + email = record["email"] + entity_tag = email_to_entity_tag(email, email_domain) + if not entity_tag: + skipped.append((email, "domain mismatch or unexpected format")) + continue + entity_series[entity_tag].append({ + "timestamp": timestamp, + "value": round(record["cost_dollars"], 2), + }) + + if not entity_series: + print("No spend records matched — nothing to push.") + else: + print(f"Pushing spend for {len(entity_series)} employee(s) to Cortex...") + push_errors = [] + for entity_tag, series in sorted(entity_series.items()): + try: + push_to_cortex(cortex_api_key, cortex_base_url, entity_tag, series) + print(f" OK: {entity_tag}") + except requests.HTTPError as e: + print(f" FAIL: {entity_tag}: {e}", file=sys.stderr) + push_errors.append(entity_tag) + + if push_errors: + print(f"\nERROR: Failed to push {len(push_errors)} entities.", file=sys.stderr) + sys.exit(1) + + # Build a flat map of employee tag → spend for rollup calculations + employee_spend = { + tag: series[0]["value"] + for tag, series in entity_series.items() + if series + } + + # Compute and push team rollups + print("\nFetching team-member relationships for team rollups...") + try: + adjacency = fetch_team_member_relationships(cortex_api_key, cortex_base_url) + except requests.HTTPError as e: + print(f"WARNING: Could not fetch team relationships, skipping rollups: {e}", file=sys.stderr) + adjacency = {} + + if adjacency: + # Identify all team nodes (source entities that have team-member relationships) + team_tags = list(adjacency.keys()) + print(f"Computing rollups for {len(team_tags)} team(s)...") + rollup_errors = [] + teams_updated = 0 + + for team_tag in sorted(team_tags): + leaf_employees = collect_leaf_employees(team_tag, adjacency) + if not leaf_employees: + continue + + team_spend = round( + sum(employee_spend.get(e, 0.0) for e in leaf_employees), 2 + ) + if team_spend == 0: + continue + + series = [{"timestamp": timestamp, "value": team_spend}] + try: + push_to_cortex(cortex_api_key, cortex_base_url, team_tag, series) + print(f" OK: {team_tag} = ${team_spend:.2f}/wk ({len(leaf_employees)} members)") + teams_updated += 1 + except requests.HTTPError as e: + print(f" FAIL: {team_tag}: {e}", file=sys.stderr) + rollup_errors.append(team_tag) + + if rollup_errors: + print(f"\nERROR: Failed to push rollups for {len(rollup_errors)} teams.", file=sys.stderr) + sys.exit(1) + else: + teams_updated = 0 + + print(f"\nSummary:") + print(f" Employees updated: {len(entity_series)}") + print(f" Teams updated: {teams_updated}") + print(f" Skipped: {len(skipped)}") + for email, reason in skipped: + print(f" - {email}: {reason}") + + +if __name__ == "__main__": + main() diff --git a/docs/superpowers/plans/2026-08-04-ai-spend-metrics.md b/docs/superpowers/plans/2026-08-04-ai-spend-metrics.md new file mode 100644 index 0000000..a773ba9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-04-ai-spend-metrics.md @@ -0,0 +1,977 @@ +# AI Usage & Spend Metrics Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add custom-metrics import support to `cortex backup import` and build the `ai-spend` solution bundle with sample entity hierarchy, sample spend data, a sync script, and a GH Actions workflow. + +**Architecture:** `backup.py` gains `_import_custom_metrics()` which reads one JSON file per metric key, groups entries by `entityTag`, and calls the per-entity bulk endpoint once per entity. The `solutions/ai-spend/` bundle is a self-contained directory that installs via `cortex backup import` and includes fictional teams/employees, 8 weeks of sample spend data, a Python sync script for the Anthropic Claude Enterprise Analytics API, and a weekly GH Actions workflow. + +**Tech Stack:** Python 3.11+, Typer, existing `CortexClient`, `requests` (sync script only) + +## Global Constraints + +- Python 3.11+ syntax only (use `int | None` union syntax, not `Optional[int]`) +- Follow existing patterns in `backup.py` exactly: `_import_*` function signature, sequential import, `(type_name, imported_count, failed_list)` return tuple +- All solution files live under `cortexapps_cli/solutions/ai-spend/` +- Metric key: `ai-spend` (must be created in Cortex UI before importing data; the bulk API does NOT auto-create definitions) +- Bulk endpoint per entity: `POST /api/v1/eng-intel/custom-metrics/{key}/entity/{tagOrId}/bulk` with body `{"series": [{"timestamp": "...", "value": N}]}` +- File format for `custom-metrics/`: one `.json` file per metric key; entries are a flat list grouped by `entityTag` in the import code +- Email → tag mapping: `first.last@` → `employee-first-last` +- Sample data: 8 weekly timestamps, every Monday 2026-06-09 through 2026-07-28 + +--- + +## File Map + +**Modified:** +- `cortexapps_cli/commands/backup.py` — add `_import_custom_metrics()`, wire into `import_tenant()` + +**Created (solution static files):** +- `cortexapps_cli/solutions/ai-spend/entity-types/employee.json` +- `cortexapps_cli/solutions/ai-spend/entity-relationship-types/team-member.json` +- `cortexapps_cli/solutions/ai-spend/catalog/team-engineering.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/team-platform.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/team-frontend.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/team-data.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/employee-alice-chen.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/employee-bob-martinez.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/employee-carol-kim.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/employee-david-osei.yaml` +- `cortexapps_cli/solutions/ai-spend/catalog/employee-emma-johnson.yaml` +- `cortexapps_cli/solutions/ai-spend/custom-metrics/ai-spend.json` +- `cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py` +- `cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml` +- `cortexapps_cli/solutions/ai-spend/README.md` + +**Tests:** +- `tests/test_backup.py` — add `test_backup_import_custom_metrics_invalid_api_key` + +--- + +## Task 1: Add `_import_custom_metrics()` to `backup.py` + +**Files:** +- Modify: `cortexapps_cli/commands/backup.py` +- Test: `tests/test_backup.py` + +**Interfaces:** +- Produces: `_import_custom_metrics(ctx, directory) -> tuple[str, int, list]` — returns `("custom-metrics", imported_count, [(file_path, error_type, error_msg), ...])` + +- [ ] **Step 1: Write the failing test** + +Add to `tests/test_backup.py`: + +```python +import json + +def test_backup_import_custom_metrics_invalid_api_key(monkeypatch): + """ + Test that backup import of custom-metrics fails cleanly with invalid API key. + """ + monkeypatch.setenv("CORTEX_API_KEY", "invalidKey") + + with tempfile.TemporaryDirectory() as tmpdir: + metrics_dir = os.path.join(tmpdir, "custom-metrics") + os.makedirs(metrics_dir) + + metric_file = os.path.join(metrics_dir, "ai-spend.json") + with open(metric_file, "w") as f: + json.dump({ + "values": [ + { + "entityTag": "employee-alice-chen", + "timestamp": "2026-07-28T00:00:00", + "value": 142.50 + } + ] + }, f) + + result = cli(["backup", "import", "-d", tmpdir], return_type=ReturnType.RAW) + assert result.exit_code != 0, ( + f"backup import should exit with non-zero code on failure, " + f"got exit_code={result.exit_code}" + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +poetry run pytest tests/test_backup.py::test_backup_import_custom_metrics_invalid_api_key -v +``` + +Expected: FAIL — `_import_custom_metrics` doesn't exist yet, but the test may actually pass vacuously (no `custom-metrics` dir handling means no failure). That's the signal: the function doesn't exist and no failure is triggered. We need to make it fail properly. + +- [ ] **Step 3: Add `_import_custom_metrics` to `backup.py`** + +Add this function after `_import_entity_relationships` (around line 472) and before `_has_relationships`: + +```python +def _import_custom_metrics(ctx, directory): + imported = 0 + failed = [] + if os.path.isdir(directory): + print("Processing: " + directory) + client = ctx.obj["client"] + for filename in sorted(os.listdir(directory)): + if not filename.endswith(".json"): + continue + file_path = os.path.join(directory, filename) + if not os.path.isfile(file_path): + continue + metric_key = filename[:-5] # strip .json + try: + print(" Importing: " + filename) + with open(file_path) as f: + data = json.load(f) + + # Group flat values list by entityTag + grouped = {} + for entry in data.get("values", []): + tag = entry["entityTag"] + if tag not in grouped: + grouped[tag] = [] + grouped[tag].append({ + "timestamp": entry["timestamp"], + "value": entry["value"], + }) + + # Call per-entity bulk endpoint once per entity + for entity_tag, series in grouped.items(): + client.post( + f"api/v1/eng-intel/custom-metrics/{metric_key}/entity/{entity_tag}/bulk", + data={"series": series}, + ) + imported += 1 + except Exception as e: + print(f" Failed to import {filename}: {type(e).__name__} - {str(e)}") + failed.append((file_path, type(e).__name__, str(e))) + return ("custom-metrics", imported, failed) +``` + +- [ ] **Step 4: Wire into `import_tenant()`** + +In `import_tenant()`, add the call after the `_import_entity_relationships` line (around line 764): + +```python + all_stats.append(_import_entity_relationships(ctx, directory + "/entity-relationships")) + all_stats.append(_import_custom_metrics(ctx, directory + "/custom-metrics")) # add this line + all_stats.append(_import_plugins(ctx, directory + "/plugins")) +``` + +- [ ] **Step 5: Add retry hint to the failure reporting block** + +In the `RETRY COMMANDS` section at the bottom of `import_tenant()`, add after the `elif import_type == "entity-relationships"` block: + +```python + elif import_type == "custom-metrics": + print(f"# Manual retry needed for custom-metrics: {file_path}") +``` + +- [ ] **Step 6: Run test to verify it passes** + +```bash +poetry run pytest tests/test_backup.py::test_backup_import_custom_metrics_invalid_api_key -v +``` + +Expected: PASS — the import now tries to call the API with an invalid key, fails, and exits non-zero. + +- [ ] **Step 7: Run full backup test suite** + +```bash +poetry run pytest tests/test_backup.py -v +``` + +Expected: all tests pass. + +- [ ] **Step 8: Commit** + +```bash +git add cortexapps_cli/commands/backup.py tests/test_backup.py +git commit -m "feat: add custom-metrics directory support to backup import" +``` + +--- + +## Task 2: Solution — Entity Types and Relationship Types + +**Files:** +- Create: `cortexapps_cli/solutions/ai-spend/entity-types/employee.json` +- Create: `cortexapps_cli/solutions/ai-spend/entity-relationship-types/team-member.json` + +**Interfaces:** +- Produces: `employee` entity type and `team-member` relationship type for use by all subsequent tasks + +- [ ] **Step 1: Create directory structure** + +```bash +mkdir -p cortexapps_cli/solutions/ai-spend/entity-types +mkdir -p cortexapps_cli/solutions/ai-spend/entity-relationship-types +mkdir -p cortexapps_cli/solutions/ai-spend/catalog +mkdir -p cortexapps_cli/solutions/ai-spend/custom-metrics +mkdir -p cortexapps_cli/solutions/ai-spend/scripts +mkdir -p cortexapps_cli/solutions/ai-spend/.github/workflows +``` + +- [ ] **Step 2: Create `entity-types/employee.json`** + +```json +{ + "type": "employee", + "name": "Employee", + "description": "A member of the organization. Used to track AI tool usage and spend per person.", + "iconTag": "Cortex-builtin::Person", + "schema": {"type": "object", "properties": {}} +} +``` + +- [ ] **Step 3: Create `entity-relationship-types/team-member.json`** + +Single relationship type that allows both `team` and `employee` as destinations, enabling full hierarchy traversal from any team node. + +```json +{ + "tag": "team-member", + "name": "Team Member", + "description": "Links a team to its direct members, which can be sub-teams or individual employees. Use this single relationship type to walk the full org hierarchy in the catalog.", + "definitionLocation": "SOURCE", + "isSingleSource": false, + "isSingleDestination": false, + "allowCycles": false, + "sourcesFilter": { + "include": true, + "types": ["team"], + "providers": [] + }, + "destinationsFilter": { + "include": true, + "types": ["team", "employee"], + "providers": [] + }, + "inheritances": [] +} +``` + +- [ ] **Step 4: Verify JSON is valid** + +```bash +python3 -c "import json; json.load(open('cortexapps_cli/solutions/ai-spend/entity-types/employee.json'))" +python3 -c "import json; json.load(open('cortexapps_cli/solutions/ai-spend/entity-relationship-types/team-member.json'))" +``` + +Expected: no output (valid JSON). + +- [ ] **Step 5: Commit** + +```bash +git add cortexapps_cli/solutions/ai-spend/entity-types/ cortexapps_cli/solutions/ai-spend/entity-relationship-types/ +git commit -m "add: ai-spend solution entity type and relationship type" +``` + +--- + +## Task 3: Solution — Catalog Entities + +**Files:** +- Create: `cortexapps_cli/solutions/ai-spend/catalog/team-engineering.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/team-platform.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/team-frontend.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/team-data.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/employee-alice-chen.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/employee-bob-martinez.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/employee-carol-kim.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/employee-david-osei.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/catalog/employee-emma-johnson.yaml` + +**Interfaces:** +- Consumes: `team-member` relationship type (Task 2) +- Produces: catalog entities with tags `team-engineering`, `team-platform`, `team-frontend`, `team-data`, `employee-alice-chen`, `employee-bob-martinez`, `employee-carol-kim`, `employee-david-osei`, `employee-emma-johnson` + +Team hierarchy: +``` +team-engineering +├── team-platform +│ ├── employee-alice-chen +│ └── employee-bob-martinez +├── team-frontend +│ ├── employee-carol-kim +│ └── employee-david-osei +└── team-data + └── employee-emma-johnson +``` + +- [ ] **Step 1: Create `catalog/team-engineering.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Engineering + x-cortex-tag: team-engineering + x-cortex-type: team + x-cortex-description: Top-level engineering organization + x-cortex-definition: {} + x-cortex-relationships: + - type: team-member + destinations: + - tag: team-platform + - tag: team-frontend + - tag: team-data +``` + +- [ ] **Step 2: Create `catalog/team-platform.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Platform + x-cortex-tag: team-platform + x-cortex-type: team + x-cortex-description: Platform engineering team + x-cortex-definition: {} + x-cortex-relationships: + - type: team-member + destinations: + - tag: employee-alice-chen + - tag: employee-bob-martinez +``` + +- [ ] **Step 3: Create `catalog/team-frontend.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Frontend + x-cortex-tag: team-frontend + x-cortex-type: team + x-cortex-description: Frontend engineering team + x-cortex-definition: {} + x-cortex-relationships: + - type: team-member + destinations: + - tag: employee-carol-kim + - tag: employee-david-osei +``` + +- [ ] **Step 4: Create `catalog/team-data.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Data + x-cortex-tag: team-data + x-cortex-type: team + x-cortex-description: Data engineering team + x-cortex-definition: {} + x-cortex-relationships: + - type: team-member + destinations: + - tag: employee-emma-johnson +``` + +- [ ] **Step 5: Create `catalog/employee-alice-chen.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Alice Chen + x-cortex-tag: employee-alice-chen + x-cortex-type: employee + x-cortex-description: Platform Engineer + x-cortex-definition: {} +``` + +- [ ] **Step 6: Create `catalog/employee-bob-martinez.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Bob Martinez + x-cortex-tag: employee-bob-martinez + x-cortex-type: employee + x-cortex-description: Platform Engineer + x-cortex-definition: {} +``` + +- [ ] **Step 7: Create `catalog/employee-carol-kim.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Carol Kim + x-cortex-tag: employee-carol-kim + x-cortex-type: employee + x-cortex-description: Frontend Engineer + x-cortex-definition: {} +``` + +- [ ] **Step 8: Create `catalog/employee-david-osei.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: David Osei + x-cortex-tag: employee-david-osei + x-cortex-type: employee + x-cortex-description: Frontend Engineer + x-cortex-definition: {} +``` + +- [ ] **Step 9: Create `catalog/employee-emma-johnson.yaml`** + +```yaml +openapi: "3.0.0" +info: + title: Emma Johnson + x-cortex-tag: employee-emma-johnson + x-cortex-type: employee + x-cortex-description: Data Engineer + x-cortex-definition: {} +``` + +- [ ] **Step 10: Verify YAML is valid** + +```bash +python3 -c " +import yaml, glob +for f in glob.glob('cortexapps_cli/solutions/ai-spend/catalog/*.yaml'): + yaml.safe_load(open(f)) + print('OK:', f) +" +``` + +Expected: `OK: ...` for all 9 files, no errors. + +- [ ] **Step 11: Commit** + +```bash +git add cortexapps_cli/solutions/ai-spend/catalog/ +git commit -m "add: ai-spend solution catalog entities — teams and employees" +``` + +--- + +## Task 4: Solution — Sample Metric Data + +**Files:** +- Create: `cortexapps_cli/solutions/ai-spend/custom-metrics/ai-spend.json` + +**Interfaces:** +- Consumes: entity tags from Task 3 +- Produces: `ai-spend.json` with 8 weekly data points per employee (format consumed by `_import_custom_metrics` from Task 1) + +8 weekly timestamps (every Monday, 2026-06-09 through 2026-07-28): +`2026-06-09T00:00:00`, `2026-06-16T00:00:00`, `2026-06-23T00:00:00`, `2026-06-30T00:00:00`, `2026-07-07T00:00:00`, `2026-07-14T00:00:00`, `2026-07-21T00:00:00`, `2026-07-28T00:00:00` + +- [ ] **Step 1: Create `custom-metrics/ai-spend.json`** + +Values are in USD dollars rounded to 2 decimal places. Each employee has varied week-to-week values so charts look natural. + +```json +{ + "values": [ + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-09T00:00:00", "value": 162.70 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-16T00:00:00", "value": 195.40 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-23T00:00:00", "value": 134.60 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-06-30T00:00:00", "value": 178.90 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-07T00:00:00", "value": 156.20 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-14T00:00:00", "value": 203.80 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-21T00:00:00", "value": 142.50 }, + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-28T00:00:00", "value": 187.30 }, + + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-09T00:00:00", "value": 83.60 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-16T00:00:00", "value": 118.20 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-23T00:00:00", "value": 91.30 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-06-30T00:00:00", "value": 103.50 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-07T00:00:00", "value": 76.80 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-14T00:00:00", "value": 112.60 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-21T00:00:00", "value": 87.20 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-28T00:00:00", "value": 98.40 }, + + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-09T00:00:00", "value": 161.40 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-16T00:00:00", "value": 149.80 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-23T00:00:00", "value": 138.20 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-06-30T00:00:00", "value": 172.60 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-07T00:00:00", "value": 155.30 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-14T00:00:00", "value": 128.40 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-21T00:00:00", "value": 167.90 }, + { "entityTag": "employee-carol-kim", "timestamp": "2026-07-28T00:00:00", "value": 143.70 }, + + { "entityTag": "employee-david-osei", "timestamp": "2026-06-09T00:00:00", "value": 63.70 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-06-16T00:00:00", "value": 89.40 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-06-23T00:00:00", "value": 58.90 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-06-30T00:00:00", "value": 71.60 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-07T00:00:00", "value": 82.10 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-14T00:00:00", "value": 54.20 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-21T00:00:00", "value": 78.40 }, + { "entityTag": "employee-david-osei", "timestamp": "2026-07-28T00:00:00", "value": 65.30 }, + + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-09T00:00:00", "value": 201.50 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-16T00:00:00", "value": 193.40 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-23T00:00:00", "value": 219.80 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-06-30T00:00:00", "value": 208.60 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-07T00:00:00", "value": 187.30 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-14T00:00:00", "value": 225.10 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-21T00:00:00", "value": 198.70 }, + { "entityTag": "employee-emma-johnson", "timestamp": "2026-07-28T00:00:00", "value": 212.40 } + ] +} +``` + +- [ ] **Step 2: Verify JSON is valid and has the right count** + +```bash +python3 -c " +import json +data = json.load(open('cortexapps_cli/solutions/ai-spend/custom-metrics/ai-spend.json')) +values = data['values'] +print(f'Total entries: {len(values)}') # expect 40 (5 employees × 8 weeks) +tags = set(v['entityTag'] for v in values) +print(f'Unique entities: {sorted(tags)}') +from collections import Counter +counts = Counter(v['entityTag'] for v in values) +print(f'Points per entity: {dict(counts)}') +" +``` + +Expected: +``` +Total entries: 40 +Unique entities: ['employee-alice-chen', 'employee-bob-martinez', 'employee-carol-kim', 'employee-david-osei', 'employee-emma-johnson'] +Points per entity: {'employee-alice-chen': 8, 'employee-bob-martinez': 8, 'employee-carol-kim': 8, 'employee-david-osei': 8, 'employee-emma-johnson': 8} +``` + +- [ ] **Step 3: Commit** + +```bash +git add cortexapps_cli/solutions/ai-spend/custom-metrics/ +git commit -m "add: ai-spend solution sample metric data (8 weeks)" +``` + +--- + +## Task 5: Solution — Sync Script + +**Files:** +- Create: `cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py` + +**Interfaces:** +- Produces: standalone Python script, callable as `python sync-claude-spend.py [--start YYYY-MM-DD] [--end YYYY-MM-DD]` +- Env vars consumed: `ANTHROPIC_ANALYTICS_KEY`, `CORTEX_API_KEY`, `CORTEX_BASE_URL`, `EMAIL_DOMAIN` + +**Note on the Anthropic endpoint:** The Claude Enterprise Analytics API costs endpoint is `GET /v1/organizations/analytics/costs`. This is based on the documented API structure; verify the exact parameters once an Analytics API key is available. The response structure follows the same pattern as the Claude Code Analytics API: `{"data": [...], "has_more": bool, "next_page": str|null}`. Cost amounts are decimal strings in cents. + +- [ ] **Step 1: Create `scripts/sync-claude-spend.py`** + +```python +#!/usr/bin/env python3 +""" +sync-claude-spend.py + +Pulls per-user spend from the Anthropic Claude Enterprise Analytics API +and pushes weekly cost data to Cortex as custom metric data points. + +Requirements: + pip install requests + +Environment variables: + ANTHROPIC_ANALYTICS_KEY Required. Analytics API key from claude.ai org settings. + Only the primary owner can create this key at: + claude.ai > Organization settings > API + CORTEX_API_KEY Required. Cortex API key. + CORTEX_BASE_URL Optional. Defaults to https://api.getcortexapp.com + EMAIL_DOMAIN Optional. Domain to strip from emails. Defaults to cortex.io + +Usage: + python sync-claude-spend.py + python sync-claude-spend.py --start 2026-07-21 --end 2026-07-28 + +Notes: + - Users who authenticate via API key (not Enterprise OAuth) will show $0 spend + in the Analytics API and are skipped automatically. + - The Cortex custom metric definition for "ai-spend" must already exist in your + Cortex instance before running this script. Create it in the Cortex UI under + Eng Intel > Custom Metrics. +""" + +import argparse +import os +import sys +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +import requests + +ANTHROPIC_BASE_URL = "https://api.anthropic.com" +ANTHROPIC_VERSION = "2023-06-01" +CORTEX_METRIC_KEY = "ai-spend" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Sync Claude Enterprise spend to Cortex custom metrics" + ) + parser.add_argument( + "--start", + help="Start date YYYY-MM-DD (default: 7 days ago)", + default=None, + ) + parser.add_argument( + "--end", + help="End date YYYY-MM-DD (default: yesterday)", + default=None, + ) + return parser.parse_args() + + +def get_env(key, required=True, default=None): + value = os.environ.get(key, default) + if required and not value: + print(f"ERROR: Environment variable {key} is required", file=sys.stderr) + sys.exit(1) + return value + + +def email_to_entity_tag(email, domain): + """ + Maps first.last@domain -> employee-first-last. + Returns None if email doesn't match the expected domain or format. + """ + if not email.endswith(f"@{domain}"): + return None + local = email.split("@")[0] + parts = local.split(".") + if len(parts) != 2: + return None + return f"employee-{parts[0]}-{parts[1]}" + + +def fetch_claude_spend(analytics_key, start_date, end_date): + """ + Fetch per-user cost data from the Claude Enterprise Analytics API. + + Returns list of dicts: {"email": str, "cost_dollars": float} + Only includes records where cost > 0. + + Endpoint: GET /v1/organizations/analytics/costs + Verify exact query parameters once an Analytics API key is available. + """ + headers = { + "x-api-key": analytics_key, + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json", + } + + results = [] + cursor = None + + while True: + params = { + "starting_at": start_date, + "ending_at": end_date, + } + if cursor: + params["page"] = cursor + + url = f"{ANTHROPIC_BASE_URL}/v1/organizations/analytics/costs" + response = requests.get(url, headers=headers, params=params, timeout=30) + response.raise_for_status() + body = response.json() + + for record in body.get("data", []): + actor = record.get("actor", {}) + email = actor.get("email_address") + if not email: + continue + + # Cost is returned as a decimal string in cents (e.g. "14250.000000" = $142.50) + cost_str = record.get("cost", "0") + try: + cost_dollars = float(cost_str) / 100 + except (ValueError, TypeError): + cost_dollars = 0.0 + + if cost_dollars > 0: + results.append({"email": email, "cost_dollars": cost_dollars}) + + if not body.get("has_more"): + break + cursor = body.get("next_page") + + return results + + +def push_to_cortex(cortex_api_key, cortex_base_url, entity_tag, series): + """ + Push spend data points for a single entity to Cortex. + + series: list of {"timestamp": str, "value": float} + Calls: POST /api/v1/eng-intel/custom-metrics/{key}/entity/{tag}/bulk + """ + url = ( + f"{cortex_base_url}/api/v1/eng-intel/custom-metrics" + f"/{CORTEX_METRIC_KEY}/entity/{entity_tag}/bulk" + ) + headers = { + "Authorization": f"Bearer {cortex_api_key}", + "Content-Type": "application/json", + } + response = requests.post( + url, headers=headers, json={"series": series}, timeout=30 + ) + response.raise_for_status() + + +def main(): + args = parse_args() + + analytics_key = get_env("ANTHROPIC_ANALYTICS_KEY") + cortex_api_key = get_env("CORTEX_API_KEY") + cortex_base_url = get_env( + "CORTEX_BASE_URL", required=False, default="https://api.getcortexapp.com" + ) + email_domain = get_env("EMAIL_DOMAIN", required=False, default="cortex.io") + + today = datetime.now(timezone.utc).date() + start_date = args.start or str(today - timedelta(days=7)) + end_date = args.end or str(today - timedelta(days=1)) + # Use end_date as the metric timestamp (represents the week ending on this date) + timestamp = f"{end_date}T00:00:00" + + print(f"Fetching Claude spend from {start_date} to {end_date}...") + + try: + spend_records = fetch_claude_spend(analytics_key, start_date, end_date) + except requests.HTTPError as e: + print(f"ERROR: Failed to fetch spend data from Anthropic: {e}", file=sys.stderr) + sys.exit(1) + + # Map emails to entity tags; collect skips + entity_series = defaultdict(list) + skipped = [] + + for record in spend_records: + email = record["email"] + entity_tag = email_to_entity_tag(email, email_domain) + if not entity_tag: + skipped.append((email, "domain mismatch or unexpected format")) + continue + entity_series[entity_tag].append({ + "timestamp": timestamp, + "value": round(record["cost_dollars"], 2), + }) + + if not entity_series: + print("No spend records matched — nothing to push.") + else: + print(f"Pushing spend for {len(entity_series)} employee(s) to Cortex...") + push_errors = [] + for entity_tag, series in sorted(entity_series.items()): + try: + push_to_cortex(cortex_api_key, cortex_base_url, entity_tag, series) + print(f" OK: {entity_tag}") + except requests.HTTPError as e: + print(f" FAIL: {entity_tag}: {e}", file=sys.stderr) + push_errors.append(entity_tag) + + if push_errors: + print(f"\nERROR: Failed to push {len(push_errors)} entities.", file=sys.stderr) + sys.exit(1) + + print(f"\nSummary:") + print(f" Updated: {len(entity_series)} employee(s)") + print(f" Skipped: {len(skipped)}") + for email, reason in skipped: + print(f" - {email}: {reason}") + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 2: Verify the script is syntactically valid** + +```bash +python3 -m py_compile cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py && echo "OK" +``` + +Expected: `OK` + +- [ ] **Step 3: Verify help output** + +```bash +python3 cortexapps_cli/solutions/ai-spend/scripts/sync-claude-spend.py --help +``` + +Expected: usage text showing `--start` and `--end` options, no errors. + +- [ ] **Step 4: Commit** + +```bash +git add cortexapps_cli/solutions/ai-spend/scripts/ +git commit -m "add: ai-spend solution sync script for Claude Enterprise Analytics API" +``` + +--- + +## Task 6: Solution — GH Actions Workflow and README + +**Files:** +- Create: `cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml` +- Create: `cortexapps_cli/solutions/ai-spend/README.md` + +**Interfaces:** +- Consumes: `scripts/sync-claude-spend.py` (Task 5) + +- [ ] **Step 1: Create `.github/workflows/sync-claude-spend.yaml`** + +```yaml +name: Sync Claude AI Spend to Cortex + +on: + schedule: + - cron: "0 6 * * 1" # Every Monday at 06:00 UTC + workflow_dispatch: # Allow manual runs from the Actions tab + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install requests + + - name: Sync Claude spend to Cortex + env: + ANTHROPIC_ANALYTICS_KEY: ${{ secrets.ANTHROPIC_ANALYTICS_KEY }} + CORTEX_API_KEY: ${{ secrets.CORTEX_API_KEY }} + run: python scripts/sync-claude-spend.py +``` + +- [ ] **Step 2: Create `README.md`** + +```markdown +# AI Spend Solution + +Track per-employee Claude AI spend in Cortex using custom metrics, with a full team +hierarchy for rollup visibility. + +## What This Installs + +| Resource | Tag / Key | +|---|---| +| Entity type | `employee` | +| Relationship type | `team-member` (team → team\|employee) | +| Teams | `team-engineering`, `team-platform`, `team-frontend`, `team-data` | +| Employees | `employee-alice-chen`, `employee-bob-martinez`, `employee-carol-kim`, `employee-david-osei`, `employee-emma-johnson` | +| Custom metric sample data | `ai-spend` (8 weeks, fictional) | + +## Prerequisites + +Before installing, create the `ai-spend` custom metric definition in your Cortex +instance: **Eng Intel → Custom Metrics → New Metric**, key: `ai-spend`. + +## Install + +```bash +cortex backup import -d /path/to/solutions/ai-spend +``` + +## Live Sync Setup + +To push real Claude spend data weekly: + +1. **Get an Analytics API key:** + - Sign in to claude.ai as the **primary owner** of your organization + - Go to **Organization settings → API** + - Enable public API access and create an Analytics API key + +2. **Add secrets to your GitHub repo:** + - `ANTHROPIC_ANALYTICS_KEY` — the Analytics API key from step 1 + - `CORTEX_API_KEY` — your Cortex API key + +3. **Copy the workflow** to your repo's `.github/workflows/` directory: + ```bash + cp .github/workflows/sync-claude-spend.yaml /.github/workflows/ + ``` + +4. **Copy the script** to your repo's `scripts/` directory: + ```bash + cp scripts/sync-claude-spend.py /scripts/ + ``` + +The workflow runs every Monday at 06:00 UTC and can be triggered manually from +the GitHub Actions tab. + +## Email → Entity Tag Mapping + +The sync script maps `first.last@yourdomain.com` → `employee-first-last`. + +Set `EMAIL_DOMAIN` in the workflow env if your domain isn't `cortex.io`: + +```yaml +env: + EMAIL_DOMAIN: yourcompany.com +``` + +## Notes + +- Users who authenticate Claude Code with a personal API key (not Enterprise OAuth) + will show $0 spend in the Analytics API and are skipped automatically. +- Cost data may take up to 24 hours to appear; query dates at least 30 days old + are considered final for billing purposes. +``` + +- [ ] **Step 3: Verify YAML workflow is valid** + +```bash +python3 -c "import yaml; yaml.safe_load(open('cortexapps_cli/solutions/ai-spend/.github/workflows/sync-claude-spend.yaml')); print('OK')" +``` + +Expected: `OK` + +- [ ] **Step 4: Commit** + +```bash +git add cortexapps_cli/solutions/ai-spend/.github/ cortexapps_cli/solutions/ai-spend/README.md +git commit -m "add: ai-spend solution GH Actions workflow and README" +``` + +--- + +## Self-Review + +### Spec coverage + +| Spec requirement | Task | +|---|---| +| `_import_custom_metrics()` in `backup.py` | Task 1 | +| Called after entity-relationships in `import_tenant()` | Task 1 | +| Filename stem = metric key | Task 1 | +| Groups by entityTag, calls per-entity bulk endpoint | Task 1 | +| Test with invalid API key | Task 1 | +| `employee` entity type | Task 2 | +| `team-member` relationship type (source=team, dest=team\|employee) | Task 2 | +| 4 teams with hierarchy | Task 3 | +| 5 employees across teams | Task 3 | +| `x-cortex-relationships` wired in team YAMLs | Task 3 | +| `custom-metrics/ai-spend.json` with 8 weeks of data | Task 4 | +| 5 employees × 8 weekly points | Task 4 | +| `sync-claude-spend.py` with Claude Enterprise Analytics API | Task 5 | +| Email → entity tag mapping | Task 5 | +| Skip $0 spend users | Task 5 | +| `ANTHROPIC_ANALYTICS_KEY`, `CORTEX_API_KEY`, `CORTEX_BASE_URL`, `EMAIL_DOMAIN` env vars | Task 5 | +| Weekly GH Actions schedule (Monday 06:00 UTC) | Task 6 | +| `workflow_dispatch` for manual runs | Task 6 | +| README with install instructions | Task 6 | + +All spec requirements covered. No gaps found. + +### Corrections from brainstorming + +- Design spec said "cross-entity bulk endpoint (no entity tag in path)" — the Cortex API has no such endpoint. Implementation correctly uses the per-entity bulk endpoint `POST .../entity/{tag}/bulk` with grouping by `entityTag` in the import code. File format is unchanged. diff --git a/docs/superpowers/specs/2026-08-04-ai-spend-metrics-design.md b/docs/superpowers/specs/2026-08-04-ai-spend-metrics-design.md new file mode 100644 index 0000000..d7ea4c8 --- /dev/null +++ b/docs/superpowers/specs/2026-08-04-ai-spend-metrics-design.md @@ -0,0 +1,197 @@ +# AI Usage & Spend Metrics — Design Spec + +**Date:** 2026-08-04 +**Linear:** CX-2 + +--- + +## Overview + +Track per-employee Claude AI spend in Cortex using custom metrics. Employees are catalog entities linked to teams via a single relationship type that supports a full team hierarchy. Spend data is pushed weekly via a script that polls the Anthropic Claude Enterprise Analytics API. + +This feature has two parts: +1. **CLI change** — add `custom-metrics` directory support to `cortex backup import` +2. **New solution** — `solutions/ai-spend/` with entity types, sample entities, sample metric data, sync script, and GH Actions workflow + +--- + +## Part 1: CLI Change — `backup import` Custom Metrics Support + +### What changes + +`cortexapps_cli/commands/backup.py` gains a new `_import_custom_metrics(ctx, directory)` function, following the exact pattern of every other `_import_*` function in that file. + +### File format + +One JSON file per metric key. Filename stem = metric key. Example: `custom-metrics/ai-spend.json` + +```json +{ + "values": [ + { "entityTag": "employee-alice-chen", "timestamp": "2026-07-28T00:00:00", "value": 142.50 }, + { "entityTag": "employee-bob-martinez", "timestamp": "2026-07-28T00:00:00", "value": 87.20 } + ] +} +``` + +### API call + +`POST /api/v1/eng-intel/custom-metrics/{key}/entity/bulk` (cross-entity bulk endpoint — no entity tag in path). + +### Import behavior + +- Reads every `*.json` file in the `custom-metrics/` directory +- Uses filename stem as the metric key +- Imports sequentially (files are independent; no ordering constraint; parallelism adds no benefit) +- Called from `import_tenant()` after `entity-relationships` so entities exist before metrics land +- Follows existing error handling and summary reporting patterns + +### What does NOT change + +- No export support — custom metrics are time-series data from external sources; backup export of them has no value +- `custom-metrics` is not added to `backupTypes` (the set used to validate the `--export-types` flag — import doesn't use it) +- No new CLI flags or commands + +--- + +## Part 2: Solution — `solutions/ai-spend/` + +### Directory structure + +``` +solutions/ai-spend/ +├── README.md +├── entity-types/ +│ └── employee.json +├── entity-relationship-types/ +│ └── team-member.json +├── catalog/ +│ ├── team-engineering.yaml +│ ├── team-platform.yaml +│ ├── team-frontend.yaml +│ ├── team-data.yaml +│ ├── employee-alice-chen.yaml +│ ├── employee-bob-martinez.yaml +│ ├── employee-carol-kim.yaml +│ ├── employee-david-osei.yaml +│ └── employee-emma-johnson.yaml +├── custom-metrics/ +│ └── ai-spend.json +├── scripts/ +│ └── sync-claude-spend.py +└── .github/ + └── workflows/ + └── sync-claude-spend.yaml +``` + +### Entity type: `employee` + +Custom entity type. Minimal schema — just enough to register the type. Icon: a person/user icon from Cortex builtins. + +### Entity relationship type: `team-member` + +- Source: `team` (built-in) +- Destination: `team` or `employee` (single type, supports both) +- This single type allows walking the full hierarchy from any team node in the catalog + +### Team hierarchy (sample data) + +``` +team-engineering (top-level) +├── team-platform +│ ├── employee-alice-chen +│ └── employee-bob-martinez +├── team-frontend +│ ├── employee-carol-kim +│ └── employee-david-osei +└── team-data + └── employee-emma-johnson +``` + +Teams are wired via `x-cortex-relationships` in the team catalog YAMLs using the `team-member` relationship type. + +### Sample metric data: `custom-metrics/ai-spend.json` + +- Metric key: `ai-spend` +- 8 weekly data points per employee, backdated from 2026-08-04 +- Timestamps: every Monday for the past 8 weeks (2026-06-09 through 2026-07-28) +- Realistic-looking fictional dollar values (range: $40–$220/week per employee), varied week-to-week so charts look natural + +### Sync script: `scripts/sync-claude-spend.py` + +**Purpose:** Pull per-user spend from the Anthropic Claude Enterprise Analytics API and push to Cortex as custom metric data points. + +**Auth context:** Cortex uses Claude Enterprise (claude.ai), not the API console. Analytics API keys are created at `claude.ai > Organization settings > API` by the primary owner. The key goes in `x-api-key` on calls to `https://api.anthropic.com/v1/organizations/analytics/`. + +**Email → entity tag mapping:** + +``` +first.last@ → employee-first-last +``` + +Domain is configurable via `EMAIL_DOMAIN` env var (default: `cortex.io`). + +**Script behavior:** +1. Compute date range: past 7 days (configurable via `--start` / `--end` flags) +2. Call Claude Enterprise Analytics API cost/usage endpoint, paginate until done +3. Filter to records where `cost > 0` (skip $0.00 users who authenticate via API key rather than Enterprise OAuth — their costs appear in separate API billing) +4. Map email to entity tag; skip records where email domain doesn't match or transform fails +5. Build `ai-spend` bulk payload and POST to Cortex custom metrics API +6. Print summary: N users updated, N skipped (with reasons) + +**Environment variables:** + +| Var | Required | Default | Description | +|-----|----------|---------|-------------| +| `ANTHROPIC_ANALYTICS_KEY` | Yes | — | Analytics API key from claude.ai org settings | +| `CORTEX_API_KEY` | Yes | — | Cortex API key | +| `CORTEX_BASE_URL` | No | `https://api.getcortexapp.com` | Cortex instance URL | +| `EMAIL_DOMAIN` | No | `cortex.io` | Domain to strip when mapping emails to entity tags | + +**Dependencies:** `requests` (no Anthropic SDK needed — Analytics API is plain REST) + +### GH Actions workflow: `.github/workflows/sync-claude-spend.yaml` + +- **Trigger:** `schedule` — weekly, every Monday at 06:00 UTC; plus `workflow_dispatch` for manual runs +- **Secrets:** `ANTHROPIC_ANALYTICS_KEY`, `CORTEX_API_KEY` +- **Steps:** checkout → `pip install requests` → run `scripts/sync-claude-spend.py` +- **Failure behavior:** non-zero exit fails the workflow so GH sends the standard failure notification + +--- + +## Data Flow + +``` +Claude Enterprise Analytics API + │ + │ GET /v1/organizations/analytics/costs + │ (weekly, per-user spend) + ▼ +sync-claude-spend.py + │ + │ email → employee-first-last + │ POST /api/v1/eng-intel/custom-metrics/ai-spend/entity/bulk + ▼ +Cortex Custom Metrics + │ + │ displayed on employee entity page + │ aggregatable up the team hierarchy + ▼ +Cortex Catalog (team-engineering → team-platform → employee-alice-chen) +``` + +--- + +## What's Not In Scope + +- Export of custom metrics (no backup export support) +- Supporting the Anthropic API console Usage & Cost API (covers API-key users with $0 enterprise spend) — deferred +- Importing custom metric *definitions* (metric key must already exist in Cortex) — the solution installs sample data but customers need to create their `ai-spend` metric definition manually or via a separate step +- Per-team spend rollup in Cortex — this is handled by Cortex natively once entity relationships are in place + +--- + +## Open Questions + +- **Custom metric definition creation:** does `backup import` need to create the metric definition (`ai-spend`) before pushing data, or does the bulk endpoint auto-create it? Needs verification against the API. +- **Analytics API endpoint:** exact endpoint path for Claude Enterprise cost-per-user report needs confirmation once an Analytics API key is available (primary owner is on sabbatical for ~8 weeks; using sample data until then). diff --git a/tests/test_backup.py b/tests/test_backup.py index be9c138..d0dcee7 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -1,6 +1,35 @@ from tests.helpers.utils import * import os import tempfile +import json + +def test_backup_import_custom_metrics_invalid_api_key(monkeypatch): + """ + Test that backup import of custom-metrics fails cleanly with invalid API key. + """ + monkeypatch.setenv("CORTEX_API_KEY", "invalidKey") + + with tempfile.TemporaryDirectory() as tmpdir: + metrics_dir = os.path.join(tmpdir, "custom-metrics") + os.makedirs(metrics_dir) + + metric_file = os.path.join(metrics_dir, "ai-spend.json") + with open(metric_file, "w") as f: + json.dump({ + "values": [ + { + "entityTag": "employee-alice-chen", + "timestamp": "2026-07-28T00:00:00", + "value": 142.50 + } + ] + }, f) + + result = cli(["backup", "import", "-d", tmpdir], return_type=ReturnType.RAW) + assert result.exit_code != 0, ( + f"backup import should exit with non-zero code on failure, " + f"got exit_code={result.exit_code}" + ) def test_backup_import_invalid_api_key(monkeypatch): """