diff --git a/AGENTS.md b/AGENTS.md index 92b6378..ca2f95e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ src/ whoami.rs Whoami command (auth check, team info) apps.rs Apps command (fetch, DB write) consumers.rs Consumers command (paginated fetch, DB write) + endpoints.rs Endpoints command (fetch, DB write) request_logs.rs Request logs command (Arrow IPC or NDJSON streaming) request_details.rs Request details command (single request fetch, DB write) sql.rs SQL command (query DuckDB, output NDJSON) @@ -44,6 +45,7 @@ skills/ | `whoami` | `GET /v1/team` | | `apps` | `GET /v1/apps` | | `consumers` | `GET /v1/apps/{app_id}/consumers` | +| `endpoints` | `GET /v1/apps/{app_id}/endpoints` | | `request-logs` | `POST /v1/apps/{app_id}/request-logs/stream` | | `request-details` | `GET /v1/apps/{app_id}/request-logs/{request_uuid}` | | `sql` | Local DuckDB | diff --git a/README.md b/README.md index a66469a..6d1fda5 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ You can also set the API key via the `APITALLY_API_KEY` environment variable or | `whoami` | Check authentication and show team info | | `apps` | List all apps in your team | | `consumers` | List consumers for an app | +| `endpoints` | List endpoints for an app | | `request-logs` | Fetch request log data for an app | | `request-details` | Fetch full details for a specific request | | `sql` | Run SQL queries against a local DuckDB database | diff --git a/skills/apitally-cli/SKILL.md b/skills/apitally-cli/SKILL.md index 4ee1949..67cad20 100644 --- a/skills/apitally-cli/SKILL.md +++ b/skills/apitally-cli/SKILL.md @@ -37,6 +37,7 @@ All commands are run via `npx @apitally/cli `. For full details, see [r - `whoami` -- check auth, show team - `apps [--db []]` -- list apps (get app IDs) - `consumers [--requests-since
] [--db []]` -- list consumers for an app (get consumer IDs) +- `endpoints [--method ] [--path ] [--db []]` -- list endpoints for an app - `request-logs --since
[--until
] [--fields ] [--filters ] [--limit ] [--db []]` -- fetch request logs (max 1,000,000 rows at once) - `request-details [--db []]` -- fetch full details for a single request (including headers, payloads, exception info, application logs, and spans) - `sql "" [--db ]` -- run SQL against local DuckDB @@ -50,18 +51,21 @@ All commands are run via `npx @apitally/cli `. For full details, see [r 3. **Determine the time range** — check if the user specified a time range (e.g. "last 24 hours", "since Monday", a specific date). If not, default to the last 7 days. Use this time range consistently for `--requests-since` / `--since` / `--until` flags and SQL `WHERE` conditions throughout the investigation. -4. **Determine if consumers are involved** — decide which scenario applies: - - **(a) Specific consumer(s)**: the user is asking about specific consumers (e.g. by email, name, or group). Fetch consumers first, then query to find the matching `consumer_id`, then use it as a filter when fetching request logs. - - **(b) Consumer context needed**: the investigation involves consumers but not specific ones known upfront (e.g. "which consumers cause the most errors"). Fetch consumers into DuckDB for later JOINs with request logs. - - **(c) No consumer involvement**: skip fetching consumers. +4. **Fetch endpoints if needed** — skip this step unless you need to discover available endpoints to filter request logs. Fetch endpoints using the `endpoints` command: -5. **Fetch consumers** into DuckDB using the `consumers` command (only if scenario (a) or (b) applies): + ``` + npx @apitally/cli endpoints [--method ] [--path ] + ``` + + Use `--method` and/or `--path` to filter (e.g. `--path '*users*'`). Read the NDJSON output to identify relevant endpoints, then use their method/path to filter request logs in step 6. + +5. **Fetch consumers if needed** — skip this step if the investigation doesn't involve consumers. Otherwise, fetch consumers into DuckDB using the `consumers` command: ``` npx @apitally/cli consumers [--requests-since ""] --db ``` - For scenario (a), query to find the consumer IDs: + If the user is asking about specific consumers (e.g. by email, name, or group), query to find their `consumer_id` and use it as a filter when fetching request logs in step 6: ``` npx @apitally/cli sql "SELECT consumer_id, identifier, name, \"group\" FROM consumers WHERE app_id = AND identifier ILIKE '%@example.com'" @@ -76,7 +80,9 @@ All commands are run via `npx @apitally/cli `. For full details, see [r --db ``` - For scenario (a), add a consumer filter: `{"field":"consumer_id","op":"in","value":[1,2,3]}` + If filtering by endpoint, add method/path filters: `[{"field":"method","op":"eq","value":"GET"},{"field":"path","op":"eq","value":"/v1/users/{user_id}"}]` + + If filtering by consumers, add a consumer filter: `[{"field":"consumer_id","op":"in","value":[1,2,3]}]` Narrow down fields and use filters as much as possible to avoid fetching unnecessarily large volumes of data. Refetching data later (e.g. with more fields) replaces existing records in DuckDB and does not create duplicates. @@ -114,21 +120,6 @@ WHERE r.app_id = ORDER BY r.timestamp DESC ``` -### Top consumers by error count - -```sql -SELECT c.identifier, c.name, - COUNT(*) as total_requests, - SUM(CASE WHEN r.status_code >= 400 THEN 1 ELSE 0 END) as errors -FROM request_logs r -JOIN consumers c ON r.app_id = c.app_id AND r.consumer_id = c.consumer_id -WHERE r.app_id = - AND r.timestamp >= '' -GROUP BY c.identifier, c.name -ORDER BY errors DESC -LIMIT 20 -``` - ### Exception investigation Fetch with exception fields first: diff --git a/skills/apitally-cli/references/commands.md b/skills/apitally-cli/references/commands.md index aac17cc..589f6db 100644 --- a/skills/apitally-cli/references/commands.md +++ b/skills/apitally-cli/references/commands.md @@ -62,6 +62,25 @@ Example NDJSON output (without `--db`): {"id":2,"identifier":"alice@example.com","name":"Alice","group":null,"created_at":"2026-01-02T00:00:00Z","last_request_at":"2026-01-02T02:00:00Z"} ``` +## `endpoints` + +``` +npx @apitally/cli endpoints [--method ] [--path ] [--db []] +``` + +List API endpoints for an app, ordered by path and method. Use this to see which endpoints exist for an app. Outputs NDJSON to stdout by default. + +- `--method`: Filter to HTTP method(s), comma-separated (e.g. `GET,POST`) +- `--path`: Filter to path pattern, supports wildcards (e.g. `/v1/*`) +- `--db`: Write to `endpoints` table in DuckDB instead of outputting NDJSON to stdout + +Example NDJSON output (without `--db`): + +```json +{"id":1,"method":"POST","path":"/v1/users"} +{"id":2,"method":"GET","path":"/v1/users/{user_id}"} +``` + ## `request-logs` ``` @@ -187,7 +206,7 @@ Run a SQL query against a local DuckDB database. The query can be passed as an a - `--db`: Path to DuckDB database -Available tables: `apps`, `app_envs`, `consumers`, `request_logs`, `application_logs`, `spans`. See [duckdb_tables.md](duckdb_tables.md) for schemas. +Available tables: `apps`, `app_envs`, `consumers`, `endpoints`, `request_logs`, `application_logs`, `spans`. See [duckdb_tables.md](duckdb_tables.md) for schemas. **Important:** The database may contain data from previous sessions. Always filter queries by `app_id`, `timestamp`, and other relevant fields to avoid including unrelated data. diff --git a/skills/apitally-cli/references/duckdb_tables.md b/skills/apitally-cli/references/duckdb_tables.md index cd56880..2d02d93 100644 --- a/skills/apitally-cli/references/duckdb_tables.md +++ b/skills/apitally-cli/references/duckdb_tables.md @@ -1,6 +1,6 @@ # DuckDB Table Schemas -Tables are created automatically when using the `--db` flag with `apps`, `consumers`, `request-logs`, or `request-details` commands. DuckDB uses a [PostgreSQL-compatible SQL dialect](https://duckdb.org/docs/stable/sql/dialect/overview). +Tables are created automatically when using the `--db` flag with `apps`, `consumers`, `endpoints`, `request-logs`, or `request-details` commands. DuckDB uses a [PostgreSQL-compatible SQL dialect](https://duckdb.org/docs/stable/sql/dialect/overview). ## apps @@ -44,6 +44,18 @@ CREATE TABLE consumers ( The `identifier` is the consumer string set in the application (e.g. email, username, API key name). The `"group"` column name is quoted because it is a reserved word in SQL. +## endpoints + +```sql +CREATE TABLE endpoints ( + app_id INTEGER NOT NULL, + endpoint_id INTEGER NOT NULL, + method TEXT NOT NULL, + path TEXT NOT NULL, + UNIQUE (app_id, endpoint_id) +); +``` + ## request_logs ```sql @@ -117,6 +129,7 @@ Populated by the `request-details` command when using `--db`. ## Relationships - `request_logs.consumer_id` references `consumers.consumer_id` (join on both `app_id` and `consumer_id`) +- `endpoints.app_id` references `apps.app_id` - `request_logs.app_id` references `apps.app_id` - `app_envs.app_id` references `apps.app_id` - `request_logs.env` matches `app_envs.name` (string, not a foreign key to `app_env_id`) diff --git a/src/apps.rs b/src/apps.rs index 3c4737c..eb4bbb5 100644 --- a/src/apps.rs +++ b/src/apps.rs @@ -106,7 +106,7 @@ pub fn run( ensure_apps_tables(&conn)?; write_apps_to_db(&conn, &apps)?; eprintln!( - "{} apps written to table 'apps' in {}...\nDone.", + "{} apps written to table 'apps' in {}.\nDone.", apps.len(), db_path.display(), ); diff --git a/src/endpoints.rs b/src/endpoints.rs new file mode 100644 index 0000000..c915b42 --- /dev/null +++ b/src/endpoints.rs @@ -0,0 +1,205 @@ +use std::io::Write; +use std::path::Path; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use crate::auth::{resolve_api_base_url, resolve_api_key}; +use crate::utils::{api_get, open_db}; + +#[derive(Deserialize)] +struct EndpointsResponse { + data: Vec, +} + +#[derive(Deserialize, Serialize)] +struct EndpointItem { + id: i64, + method: String, + path: String, +} + +fn fetch_endpoints( + api_key: &str, + api_base_url: &str, + app_id: i64, + method: Option<&str>, + path: Option<&str>, +) -> Result> { + let url = format!("{api_base_url}/v1/apps/{app_id}/endpoints"); + let mut query: Vec<(&str, &str)> = Vec::new(); + if let Some(m) = method { + query.push(("method", m)); + } + if let Some(p) = path { + query.push(("path", p)); + } + let mut response = api_get(&url, api_key, &query)?; + let endpoints: EndpointsResponse = response.body_mut().read_json()?; + Ok(endpoints.data) +} + +pub(crate) fn ensure_endpoints_table(conn: &duckdb::Connection) -> Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS endpoints ( + app_id INTEGER NOT NULL, + endpoint_id INTEGER NOT NULL, + method TEXT NOT NULL, + path TEXT NOT NULL, + UNIQUE (app_id, endpoint_id) + )", + )?; + Ok(()) +} + +fn write_endpoints_to_db( + conn: &duckdb::Connection, + app_id: i64, + endpoints: &[EndpointItem], +) -> Result<()> { + let mut stmt = conn.prepare( + "INSERT OR REPLACE INTO endpoints ( + app_id, endpoint_id, method, path + ) VALUES (?, ?, ?, ?)", + )?; + for endpoint in endpoints { + stmt.execute(duckdb::params![ + app_id, + endpoint.id, + endpoint.method, + endpoint.path, + ])?; + } + Ok(()) +} + +pub fn run( + app_id: i64, + method: Option<&str>, + path: Option<&str>, + db: Option<&Path>, + api_key: Option<&str>, + api_base_url: Option<&str>, + mut writer: impl Write, +) -> Result<()> { + let api_key = resolve_api_key(api_key)?; + let api_base_url = resolve_api_base_url(api_base_url); + let endpoints = fetch_endpoints(&api_key, &api_base_url, app_id, method, path)?; + + if let Some(db_path) = db { + let conn = open_db(db_path)?; + ensure_endpoints_table(&conn)?; + write_endpoints_to_db(&conn, app_id, &endpoints)?; + eprintln!( + "{} endpoints written to table 'endpoints' in {}.\nDone.", + endpoints.len(), + db_path.display(), + ); + } else { + for endpoint in &endpoints { + serde_json::to_writer(&mut writer, endpoint)?; + writeln!(writer)?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utils::open_db; + use crate::utils::test_utils::{parse_ndjson, temp_db}; + + fn sample_endpoints_json() -> &'static str { + r#"{ + "data": [ + { + "id": 1, + "method": "POST", + "path": "/v1/users" + }, + { + "id": 2, + "method": "GET", + "path": "/v1/users/{user_id}" + } + ] + }"# + } + + fn mock_endpoints_endpoint(server: &mut mockito::Server, app_id: i64) -> mockito::Mock { + let path = format!("/v1/apps/{app_id}/endpoints"); + server + .mock("GET", path.as_str()) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(sample_endpoints_json()) + .create() + } + + #[test] + fn test_run_ndjson() { + let mut server = mockito::Server::new(); + let mock = mock_endpoints_endpoint(&mut server, 1); + + let mut buf = Vec::new(); + run( + 1, + None, + None, + None, + Some("test-key"), + Some(&server.url()), + &mut buf, + ) + .unwrap(); + mock.assert(); + + let rows = parse_ndjson(&buf); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0]["method"], "POST"); + assert_eq!(rows[0]["path"], "/v1/users"); + assert_eq!(rows[1]["method"], "GET"); + assert_eq!(rows[1]["path"], "/v1/users/{user_id}"); + } + + #[test] + fn test_run_with_db() { + let mut server = mockito::Server::new(); + let mock = mock_endpoints_endpoint(&mut server, 1); + let (_dir, db_path) = temp_db(); + + run( + 1, + None, + None, + Some(&db_path), + Some("test-key"), + Some(&server.url()), + Vec::new(), + ) + .unwrap(); + mock.assert(); + + let conn = open_db(&db_path).unwrap(); + + let count: i64 = conn + .query_row( + "SELECT count(*) FROM endpoints WHERE app_id = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 2); + + let method: String = conn + .query_row( + "SELECT method FROM endpoints WHERE endpoint_id = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(method, "POST"); + } +} diff --git a/src/main.rs b/src/main.rs index 0142820..65f4366 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ mod apps; mod auth; mod consumers; +mod endpoints; mod request_details; mod request_logs; mod reset_db; @@ -90,6 +91,32 @@ enum Command { db: Option>, }, + /// List endpoints for an app + /// + /// Outputs newline-delimited JSON (one object per line). + /// With --db, inserts rows into the `endpoints` table instead. + Endpoints { + #[command(flatten)] + api: ApiArgs, + + /// App ID + app_id: i64, + + /// Filter to HTTP method(s), comma-separated + #[arg(long)] + method: Option, + + /// Filter to path pattern, supports wildcards (*) + #[arg(long)] + path: Option, + + /// Store results in DuckDB instead of outputting NDJSON + /// + /// Defaults to ~/.apitally/data.duckdb if no path is given. + #[arg(long, num_args = 0..=1)] + db: Option>, + }, + /// Retrieve request log data for an app /// /// Outputs newline-delimited JSON (one object per line). @@ -181,7 +208,7 @@ enum Command { /// Run a SQL query against local DuckDB /// - /// Available tables: apps, app_envs, consumers, request_logs, + /// Available tables: apps, app_envs, consumers, endpoints, request_logs, /// application_logs, spans. Sql { /// SQL query to execute (reads from stdin if omitted) @@ -274,6 +301,24 @@ fn run(cli: Cli) -> Result<()> { std::io::stdout().lock(), ) } + Command::Endpoints { + api, + app_id, + method, + path, + db, + } => { + let db = utils::resolve_db(db)?; + endpoints::run( + app_id, + method.as_deref(), + path.as_deref(), + db.as_deref(), + api.api_key.as_deref(), + api.api_base_url.as_deref(), + std::io::stdout().lock(), + ) + } Command::RequestLogs { api, app_id, @@ -353,6 +398,7 @@ mod tests { // Missing required args should fail assert!(Cli::try_parse_from(["apitally"]).is_err()); // missing command assert!(Cli::try_parse_from(["apitally", "consumers"]).is_err()); // missing app_id + assert!(Cli::try_parse_from(["apitally", "endpoints"]).is_err()); // missing app_id assert!(Cli::try_parse_from(["apitally", "request-logs", "42"]).is_err()); // missing --since assert!(Cli::try_parse_from(["apitally", "request-details", "42"]).is_err()); // missing request_uuid assert!(Cli::try_parse_from(["apitally", "sql", "SELECT 1", "--db"]).is_err()); // missing db path @@ -376,6 +422,12 @@ mod tests { .command, Command::Consumers { app_id: 42, .. } )); + assert!(matches!( + Cli::try_parse_from(["apitally", "endpoints", "42"]) + .unwrap() + .command, + Command::Endpoints { app_id: 42, .. } + )); assert!(matches!( Cli::try_parse_from(["apitally", "request-logs", "42", "--since", "2025-01-01"]) .unwrap() diff --git a/src/reset_db.rs b/src/reset_db.rs index 0f83921..bd4c6bf 100644 --- a/src/reset_db.rs +++ b/src/reset_db.rs @@ -3,7 +3,7 @@ use std::path::Path; use anyhow::Result; use crate::utils::open_db; -use crate::{apps, consumers, request_details, request_logs}; +use crate::{apps, consumers, endpoints, request_details, request_logs}; pub fn run(db: &Path) -> Result<()> { let conn = open_db(db)?; @@ -19,6 +19,7 @@ pub fn run(db: &Path) -> Result<()> { apps::ensure_apps_tables(&conn)?; consumers::ensure_consumers_table(&conn)?; + endpoints::ensure_endpoints_table(&conn)?; request_logs::ensure_request_logs_table(&conn)?; request_details::ensure_application_logs_table(&conn)?; request_details::ensure_spans_table(&conn)?; @@ -59,6 +60,7 @@ mod tests { "application_logs", "apps", "consumers", + "endpoints", "request_logs", "spans" ]