Add discover: a breadth-first walk of the page graph

cli/discover.go

@@ -0,0 +1,168 @@
+package cli
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/tamnd/wikipedia-cli/wiki"
+)
+
+// defaultDiscoverBudget caps a streaming walk when -n is not given, so
+// `wiki discover <page>` always terminates instead of spidering the wiki forever.
+const defaultDiscoverBudget = 500
+
+// newDiscoverCmd is the breadth-first graph walk. Where each structure command
+// answers one question about one page (its links, its categories, a category's
+// members), discover chains them: from a seed page or category it follows the
+// object's links and from each neighbor it follows theirs, hop by hop, streaming
+// one row per node as it is reached.
+func newDiscoverCmd(app *App) *cobra.Command {
+	var (
+		depth  int
+		fanout int
+		follow string
+	)
+	cmd := &cobra.Command{
+		Use:     "discover <seed>...",
+		Aliases: []string{"walk", "graph"},
+		Short:   "Breadth-first walk of the graph linked from a page or category",
+		Long: `Walk the graph of linked Wikipedia objects, breadth first, starting from one
+or more seeds. A seed is anything wiki can resolve: an article title or URL, or
+a category name or URL.
+
+--follow chooses which links to traverse. It takes a preset or a comma-separated
+edge list:
+
+  content   a page's links and categories, a category's members and
+            subcategories (the default; the obvious neighbors)
+  network   a page's outgoing links and its backlinks
+  cats      a page's categories, and a category's members and subcategories
+  all       every edge
+
+Edges: links, backlinks, categories, members, subcats. Name an edge directly to
+follow just that link, e.g. --follow backlinks to walk what-links-here.
+
+--depth is how many hops to follow (default 1; 0 emits only the seeds). --fanout
+caps neighbors per edge (default 25). The walk streams nodes and stops after -n
+nodes (default 500).
+
+wiki keeps no local database, so discover streams to stdout. To keep a walk,
+pipe it: wiki discover "Alan Turing" --depth 2 -o jsonl > turing-graph.jsonl.`,
+		Args: cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			edges, err := wiki.ParseEdges(follow)
+			if err != nil {
+				return usageErr(err.Error())
+			}
+			seeds, err := parseSeeds(args)
+			if err != nil {
+				return err
+			}
+			c, err := app.Client()
+			if err != nil {
+				return err
+			}
+
+			budget := app.Limit
+			if budget <= 0 {
+				budget = defaultDiscoverBudget
+			}
+
+			opts := wiki.WalkOptions{
+				Depth:  depth,
+				Max:    budget,
+				Fanout: fanout,
+				Edges:  edges,
+				Note: func(s string) {
+					if !app.quiet {
+						fmt.Fprintf(os.Stderr, "wiki: note: %s\n", s)
+					}
+				},
+			}
+
+			sp := app.progress("walking")
+			n := 0
+			walkErr := c.Walk(cmd.Context(), seeds, opts, func(nd *wiki.Node) error {
+				sp.stop() // clear the spinner before the first row reaches stdout
+				n++
+				return app.Out.Emit(nodeRow(nd))
+			})
+			sp.stop()
+			flushErr := app.Out.Flush()
+			if walkErr != nil {
+				// The only fatal walk error is a seed that could not be fetched;
+				// it surfaces like a failed single read. Deeper failures are notes.
+				return wrapErr(walkErr)
+			}
+			if flushErr != nil {
+				return flushErr
+			}
+			if n == 0 {
+				return noResults("nothing discovered")
+			}
+			return nil
+		},
+	}
+	cmd.Flags().IntVar(&depth, "depth", 1, "hops to follow from each seed (0 = seeds only)")
+	cmd.Flags().IntVar(&fanout, "fanout", 25, "max neighbors to follow per edge (0 = unlimited)")
+	cmd.Flags().StringVar(&follow, "follow", "content", "edges to follow ("+wiki.EdgeHelp()+")")
+	return cmd
+}
+
+// parseSeeds turns the positional arguments into walk seeds, reporting an
+// unrecognized reference as a usage error rather than a plain failure.
+func parseSeeds(args []string) ([]wiki.Seed, error) {
+	seeds := make([]wiki.Seed, 0, len(args))
+	for _, a := range args {
+		s, err := wiki.ParseSeed(a)
+		if err != nil {
+			return nil, usageErr(err.Error())
+		}
+		seeds = append(seeds, s)
+	}
+	return seeds, nil
+}
+
+// nodeRow renders a graph node discovered by `wiki discover`. The curated columns
+// read the walk at a glance (how deep, by which edge, the title and a one-line
+// gloss) while the full typed node rides in Value for json, jsonl, and templates.
+func nodeRow(n *wiki.Node) Row {
+	var title, summary, url string
+	switch n.Kind {
+	case wiki.KindPage:
+		p := n.Page
+		title = p.Title
+		summary = oneline(firstNonEmpty(p.Description, p.Extract))
+		url = p.URL
+	case wiki.KindCategory:
+		c := n.Category
+		title = c.Title
+		url = c.URL
+	}
+	return Row{
+		Cols:  []string{"depth", "via", "kind", "title", "summary", "url"},
+		Vals:  []string{itoa(n.Depth), string(n.Via), string(n.Kind), title, summary, url},
+		Value: n,
+	}
+}
+
+// oneline flattens a value to a single short line for a curated column, so a
+// multi-line extract never breaks the list and table layouts.
+func oneline(s string) string {
+	s = strings.Join(strings.Fields(s), " ")
+	if len(s) > 80 {
+		return s[:79] + "..."
+	}
+	return s
+}
+
+func firstNonEmpty(vals ...string) string {
+	for _, v := range vals {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}

cli/root.go

@@ -121,6 +121,7 @@ Quick start:
 		newBacklinksCmd(app),
 		newCategoriesCmd(app),
 		newCategoryCmd(app),
+		newDiscoverCmd(app),
 		newMediaCmd(app),
 		newReferencesCmd(app),
 		newCiteCmd(app),

docs/content/guides/_index.md

@@ -16,6 +16,8 @@ depend on each other.
   random, and related pages.
 - [Page structure](/guides/structure/): links, backlinks, categories, media,
   references, languages, info, and citations.
+- [Discovering](/guides/discovering/): a breadth-first `discover` walk that
+  chains those edges across pages and categories.
 - [History and diffs](/guides/history/): revisions and unified diffs.
 - [Feeds and metrics](/guides/feeds-and-metrics/): featured, on this day, top,
   and pageviews.

docs/content/guides/discovering.md

@@ -0,0 +1,133 @@
+---
+title: "Discovering"
+description: "Walk the graph of pages and categories breadth first, following links, backlinks, categories, members, and subcategories, one record per node."
+weight: 4
+---
+
+Every structure command answers one question about one object: a page's links,
+its categories, the members of a category. `discover` chains them. From a seed it
+follows the object's links outward, and from each neighbor it follows theirs, hop
+by hop, streaming one record per node as the node is reached.
+
+```bash
+wiki discover "Alan Turing"
+```
+
+A seed is anything `wiki` can resolve: an article title or URL, or a category
+name or URL. Pass several to walk from all of them at once.
+
+## The graph
+
+There are two kinds of node, and five edges between them:
+
+| Kind | What it is |
+|---|---|
+| `page` | an article |
+| `category` | a category |
+
+| Edge | From to | What it follows |
+|---|---|---|
+| `links` | page to page | the article's outgoing internal links |
+| `backlinks` | page to page | what links here |
+| `categories` | page to category | the categories the article belongs to |
+| `members` | category to page | the articles in the category |
+| `subcats` | category to category | the category's subcategories |
+
+You rarely name edges one at a time. `--follow` takes a **preset**:
+
+| Preset | Expands to | Walk shape |
+|---|---|---|
+| `content` *(default)* | `links` + `categories` + `members` + `subcats` | the obvious forward neighbors: a page's links and categories, a category's members and subcategories |
+| `network` | `links` + `backlinks` | a page's outgoing links and what links back to it |
+| `cats` | `categories` + `members` + `subcats` | the category tree above and below a page |
+| `all` | every edge | the whole reachable neighborhood |
+
+```bash
+wiki discover "Alan Turing"                        # content (the default)
+wiki discover "Alan Turing" --follow network       # links and backlinks
+wiki discover "Category:Computer scientists" --follow cats --depth 2
+wiki discover "Alan Turing" --follow all --depth 2
+```
+
+`--follow` also takes a single edge name, or a comma-separated mix of presets and
+edges, so you can be exact:
+
+```bash
+wiki discover "Alan Turing" --follow backlinks     # only what-links-here
+wiki discover "Alan Turing" --follow links,categories
+```
+
+Preset names and edge names are deliberately disjoint, so no `--follow` token is
+ever ambiguous.
+
+## Bounding the walk
+
+Three independent limits keep a walk finite, so an unbounded `discover` always
+terminates instead of spidering the whole wiki:
+
+- `--depth` is how many hops to follow (default `1`; `0` emits only the seeds).
+- `--fanout` caps neighbors per edge (default `25`; `0` means unlimited).
+- `-n` caps the total nodes streamed (default `500`).
+
+```bash
+wiki discover "Alan Turing" --depth 2 --fanout 10 -n 200
+```
+
+A page links to hundreds of others, so a deep walk fans out fast. Raise `--depth`
+one hop at a time and lean on `--fanout` and `-n` to keep a walk the size you want.
+
+## Reading the output
+
+Each row is a node tagged with how it was reached: how deep, by which edge, the
+title and a one-line gloss. The first row is the seed at depth 0; the rest are
+its neighbors, each tagged with the edge it was reached by:
+
+```
+depth  via         kind      title
+0                  page      Alan Turing
+1      links       page      Turing machine
+1      categories  category  1912 births
+```
+
+The full typed record rides along for `-o json` and `-o jsonl`, and `-o url`
+prints one link per node:
+
+```bash
+wiki discover "Alan Turing"             # the readable table
+wiki discover "Alan Turing" -o jsonl    # one lossless object per line
+wiki discover "Alan Turing" -o url      # one URL per node, to pipe onward
+```
+
+`wiki` keeps no local database, so `discover` streams to stdout. To keep a walk,
+redirect it, and reshape it with ordinary tools:
+
+```bash
+wiki discover "Alan Turing" --depth 2 -o jsonl > turing-graph.jsonl
+
+# the distinct article titles two hops out
+wiki discover "Pi" --depth 2 -o jsonl \
+  | jq -r 'select(.kind=="page").page.title' | sort -u
+```
+
+## When an edge is gated
+
+Wikipedia's API is uniformly open: there are no scrape tiers and no per-IP
+content gates, so every edge is reachable. The only runtime friction is rate
+limiting, which the HTTP client absorbs with backoff. The walk still treats a
+failure at the seed differently from one deeper in:
+
+- A **seed** that cannot be fetched fails the walk, like any failed single read.
+- An edge that fails **deeper** in the walk becomes a one-line note on stderr
+  (`wiki: note: ...`) and the walk carries on with the other edges. `-q`
+  silences the notes.
+
+## Discover or the structure commands?
+
+`discover` does not replace the focused reads, it composes them. Reach for the
+single-purpose command when you want one slice of one page; reach for `discover`
+when you want that slice and what it links to, hop after hop.
+
+- [`links`](/guides/structure/), `backlinks`, `categories`, and the category
+  reads each return one edge of one object, exactly and completely.
+- `discover` follows those same edges outward across many objects, deduping as it
+  goes, and streams the result as one graph.

docs/content/guides/dumps.md

@@ -1,7 +1,7 @@
 ---
 title: "Dumps"
 description: "List, download, and stream-parse the public Wikimedia XML dumps with resume, sha1 verification, and constant memory."
-weight: 8
+weight: 9
 ---
 
 When you need the whole encyclopedia rather than one page, the public XML dumps

docs/content/guides/feeds-and-metrics.md

@@ -1,7 +1,7 @@
 ---
 title: "Feeds and metrics"
 description: "Browse the daily featured feed and on-this-day events, list the most-viewed articles, and chart pageviews over time."
-weight: 5
+weight: 6
 ---
 
 ## The featured feed

docs/content/guides/geo.md

@@ -1,7 +1,7 @@
 ---
 title: "Geo"
 description: "Find articles near a coordinate or near another article."
-weight: 6
+weight: 7
 ---
 
 Many Wikipedia articles carry coordinates. wiki lets you search by them.

docs/content/guides/history.md

@@ -1,7 +1,7 @@
 ---
 title: "History and diffs"
 description: "Walk a page's revision history and compare any two revisions as a unified diff."
-weight: 4
+weight: 5
 ---
 
 ## Revision history

docs/content/guides/resource-uris.md

@@ -1,7 +1,7 @@
 ---
 title: "Resource URIs"
 description: "Use wiki as a database/sql-style driver so a host program can address Wikipedia pages and categories as wikipedia:// URIs."
-weight: 9
+weight: 10
 ---
 
 `wiki` is a command line, but the `wiki` Go package is also a small driver that

docs/content/guides/structure.md

@@ -4,7 +4,8 @@ description: "Explore the graph around a page: links, backlinks, categories and
 weight: 3
 ---
 
-Every article is a node in a graph. These commands walk the edges.
+Every article is a node in a graph. These commands walk the edges, one edge at a
+time; [`discover`](/guides/discovering/) walks many at once.
 
 ## Links and backlinks
 
@@ -91,3 +92,15 @@ wiki cite "Alan Turing" --format ris
 wiki cite "Alan Turing" --format apa
 wiki cite "Alan Turing" --format mla
 ```
+
+## Discover: walk the graph
+
+Each command above answers one question about one page. `discover` chains them:
+from a seed it follows the page's edges, then each neighbor's edges, breadth
+first, streaming one row per object it reaches. The [Discovering](/guides/discovering/)
+guide covers the walk in full: the edge presets, the depth and fanout bounds, and
+how the output reads.
+
+```bash
+wiki discover "Alan Turing"              # the obvious neighbors, one hop out
+```

docs/content/guides/wikidata.md

@@ -1,7 +1,7 @@
 ---
 title: "Wikidata"
 description: "Look up structured entities by id or article title, and run raw SPARQL against the Wikidata Query Service."
-weight: 7
+weight: 8
 ---
 
 Wikidata is the structured knowledge graph behind Wikipedia. wiki reads it two