{ "widgets": [ { "type": "text", "x": 0, "y": 0, "width": 24, "height": 2, "properties": { "markdown": "## Overview", "background": "transparent" } }, { "type": "chart", "x": 0, "y": 2, "width": 6, "height": 4, "properties": { "title": "Total Tokens", "view": "number", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum(histogram_sum({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "Tokens" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": true }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 6, "y": 2, "width": 6, "height": 4, "properties": { "title": "Sessions (VS Code)", "view": "number", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum({\"copilot_chat.session.count\", \"@resource.service.name\"=~\"copilot.*\"})", "step": 60, "label": "Sessions" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": true }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 12, "y": 2, "width": 6, "height": 4, "properties": { "title": "Active Users", "view": "number", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "count(group by (\"@resource.user.email\")({\"copilot_chat.session.count\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "Users" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": true }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 18, "y": 2, "width": 6, "height": 4, "properties": { "title": "Tool Calls", "view": "number", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum({\"copilot_chat.tool.call.count\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.tool.call.count\", \"@resource.service.name\"=~\"copilot.*\"})", "step": 60, "label": "Tool Calls" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": true }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "text", "x": 0, "y": 6, "width": 24, "height": 2, "properties": { "markdown": "## Token Usage", "background": "transparent" } }, { "type": "chart", "x": 0, "y": 8, "width": 12, "height": 6, "properties": { "title": "Token Usage Over Time", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum(histogram_sum({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "Tokens" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 12, "y": 8, "width": 12, "height": 6, "properties": { "title": "Token Usage by Type", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "input", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum(histogram_sum({\"gen_ai.client.token.usage\", \"gen_ai.token.type\"=\"input\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "Input" }, { "id": "output", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum(histogram_sum({\"gen_ai.client.token.usage\", \"gen_ai.token.type\"=\"output\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "Output" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 0, "y": 14, "width": 12, "height": 6, "properties": { "title": "Token Usage by Model", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, histogram_sum(sum by (\"gen_ai.request.model\")({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"})))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": true, "stacked": true, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 12, "y": 14, "width": 12, "height": 6, "properties": { "title": "Top Users by Token Usage", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, histogram_sum(sum by (\"@resource.user.email\")({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"})))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "text", "x": 0, "y": 20, "width": 24, "height": 2, "properties": { "markdown": "## Performance & Latency", "background": "transparent" } }, { "type": "chart", "x": 0, "y": 22, "width": 8, "height": 6, "properties": { "title": "LLM Operation Duration (p50/p90/p99 s)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "p50", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.5, sum({\"gen_ai.client.operation.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p50" }, { "id": "p90", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.9, sum({\"gen_ai.client.operation.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p90" }, { "id": "p99", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.99, sum({\"gen_ai.client.operation.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p99" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 8, "y": 22, "width": 8, "height": 6, "properties": { "title": "Time to First Token (p50/p90/p99 s)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "p50", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.5, sum({\"copilot_chat.time_to_first_token\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p50" }, { "id": "p90", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.9, sum({\"copilot_chat.time_to_first_token\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p90" }, { "id": "p99", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.99, sum({\"copilot_chat.time_to_first_token\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p99" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 16, "y": 22, "width": 8, "height": 6, "properties": { "title": "LLM Latency p90 by Model (s)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.9, sum by (\"gen_ai.request.model\")({\"gen_ai.client.operation.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 0, "y": 28, "width": 8, "height": 6, "properties": { "title": "Tool Call Latency (p50/p90/p99 s)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "p50", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.5, sum({\"copilot_chat.tool.call.duration\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.tool.call.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p50" }, { "id": "p90", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.9, sum({\"copilot_chat.tool.call.duration\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.tool.call.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p90" }, { "id": "p99", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.99, sum({\"copilot_chat.tool.call.duration\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.tool.call.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p99" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 8, "y": 28, "width": 8, "height": 6, "properties": { "title": "Agent Invocation Duration (p90 s, VS Code)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.9, sum({\"copilot_chat.agent.invocation.duration\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p90" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 16, "y": 28, "width": 8, "height": 6, "properties": { "title": "Agent Turns per Invocation (p90)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "histogram_quantile(0.9, sum({\"copilot_chat.agent.turn.count\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.agent.turn.count\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60, "label": "p90" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "text", "x": 0, "y": 34, "width": 24, "height": 2, "properties": { "markdown": "## Tool & Developer Activity", "background": "transparent" } }, { "type": "chart", "x": 0, "y": 36, "width": 8, "height": 6, "properties": { "title": "Tool Calls by Tool", "view": "pie", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, sum by (\"gen_ai.tool.name\")({\"copilot_chat.tool.call.count\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.tool.call.count\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 8, "y": 36, "width": 8, "height": 6, "properties": { "title": "Tool Call Outcomes", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, sum by (success)({\"copilot_chat.tool.call.count\", \"@resource.service.name\"=~\"copilot.*\"} or {\"github.copilot.tool.call.count\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 16, "y": 36, "width": 8, "height": 6, "properties": { "title": "Lines of Code (total, VS Code)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum({\"copilot_chat.lines_of_code.count\", \"@resource.service.name\"=~\"copilot.*\"})", "step": 60, "label": "Lines" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 0, "y": 42, "width": 8, "height": 6, "properties": { "title": "Edit Acceptance Decisions (total, VS Code)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum({\"copilot_chat.edit.acceptance.count\", \"@resource.service.name\"=~\"copilot.*\"})", "step": 60, "label": "Edits" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 8, "y": 42, "width": 8, "height": 6, "properties": { "title": "User Feedback (total, VS Code)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum({\"copilot_chat.user.feedback.count\", \"@resource.service.name\"=~\"copilot.*\"})", "step": 60, "label": "Feedback" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 16, "y": 42, "width": 8, "height": 6, "properties": { "title": "Pull Requests (VS Code)", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "sum({\"copilot_chat.pull_request.count\", \"@resource.service.name\"=~\"copilot.*\"})", "step": 60, "label": "PRs" } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": false, "stacked": false, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "text", "x": 0, "y": 48, "width": 24, "height": 2, "properties": { "markdown": "## Organizational Breakdown", "background": "transparent" } }, { "type": "chart", "x": 0, "y": 50, "width": 12, "height": 6, "properties": { "title": "Token Usage by Department", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, histogram_sum(sum by (\"@resource.department\")({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"})))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": true, "stacked": true, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 12, "y": 50, "width": 12, "height": 6, "properties": { "title": "Token Usage by Team", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, histogram_sum(sum by (\"@resource.team.id\")({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"})))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": true, "stacked": true, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 0, "y": 56, "width": 12, "height": 6, "properties": { "title": "Token Usage by Cost Center", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, histogram_sum(sum by (\"@resource.cost_center\")({\"gen_ai.client.token.usage\", \"@resource.service.name\"=~\"copilot.*\"})))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": true, "stacked": true, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } }, { "type": "chart", "x": 12, "y": 56, "width": 12, "height": 6, "properties": { "title": "Sessions by Environment", "view": "line", "region": "us-east-1", "data": { "queries": [ { "id": "a", "type": "cloudwatch-metrics", "language": "PromQL", "query": "topk(500, sum by (\"@resource.environment\")({\"copilot_chat.session.count\", \"@resource.service.name\"=~\"copilot.*\"}))", "step": 60 } ] }, "plotOptions": { "legend": { "position": "bottom", "show": true }, "xAxis": { "type": "datetime" }, "yAxis": [ { "type": "linear" } ], "style": { "lineOptions": { "filled": true, "stacked": true, "width": 2, "pattern": "solid", "spline": false }, "numberOptions": { "sparkline": false }, "pieOptions": { "innerSize": "50%" }, "barOptions": {}, "gaugeOptions": {} } } } } ] }