Add discover, a breadth-first walk of the YouTube graph

cli/commands.go

@@ -16,6 +16,7 @@ func registerEscapeHatches(app *kit.App) {
 	app.AddCommand(newDownloadCmd())
 	app.AddCommand(newExtractCmd())
 	app.AddCommand(newMusicCmd())
+	app.AddCommand(newDiscoverCmd())
 	app.AddCommand(newSeedCmd())
 	app.AddCommand(newCrawlCmd())
 	app.AddCommand(newQueueCmd())

cli/discover.go

@@ -0,0 +1,145 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/tamnd/any-cli/kit"
+	"github.com/tamnd/ytb-cli/youtube"
+)
+
+// defaultDiscoverBudget caps a streaming walk when the user did not pass -n, so
+// `ytb discover <video>` always terminates instead of spidering YouTube forever.
+const defaultDiscoverBudget = 500
+
+// newDiscoverCmd is the breadth-first graph walk. Where the record reads each
+// answer one question about one object, discover chains them: from a seed video,
+// channel, or playlist it follows the object's links and from each neighbor it
+// follows theirs, hop by hop, emitting one row per node as it is reached.
+//
+// It shares the read group with the per-object commands because it is a read; it
+// only touches the store when --store is set, where it persists each node and
+// records every traversed edge into the edges table.
+func newDiscoverCmd() kit.Command {
+	var (
+		depth  int
+		fanout int
+		follow string
+		store  bool
+	)
+	return kit.Command{
+		Use:     "discover <seed>...",
+		Aliases: []string{"walk", "graph"},
+		Group:   "read",
+		Short:   "Breadth-first walk of the graph linked from a video, channel, or playlist",
+		Long: `Walk the graph of linked YouTube objects, breadth first, starting from one or
+more seeds. A seed is anything ytb can resolve: a video id or watch URL, a
+channel id, @handle, or URL, or a playlist id or URL.
+
+--follow chooses which links to traverse. It takes a preset or a comma-separated
+edge list:
+
+  content   a video's channel and related videos, a channel's uploads, a
+            playlist's items and owner (the default; the obvious neighbors)
+  feed      a channel's uploads, playlists, and community posts
+  comments  a video's comments and the channels that wrote them
+  all       every edge
+
+Edges: channel, related, comments, uploads, playlists, community, items, owner,
+commenter. Name an edge directly to follow just that link, e.g. --follow related
+to chase the recommendation graph.
+
+--depth is how many hops to follow (default 1; 0 emits only the seeds). --fanout
+caps neighbors per edge (default 25). The walk streams nodes and stops after -n
+nodes (default 500). Comments are served only when YouTube is not applying its
+per-IP Restricted Mode to this network; when it is, the comment edges are noted
+and skipped and the rest of the walk continues.
+
+Add --store to persist every node into its typed table and record each traversed
+edge into the edges table, so a walk doubles as a crawl. Query it afterwards with
+ytb db query.`,
+		Args: kit.MinimumNArgs(1),
+		Flags: func(f *kit.FlagSet) {
+			f.IntVar(&depth, "depth", 1, "hops to follow from each seed (0 = seeds only)")
+			f.IntVar(&fanout, "fanout", 25, "max neighbors to follow per edge (0 = unlimited)")
+			f.StringVar(&follow, "follow", "content", "edges to follow ("+youtube.EdgeHelp()+")")
+			f.BoolVar(&store, "store", false, "persist nodes and edges into the local store")
+		},
+		Run: func(ctx context.Context, args []string) error {
+			app := appFromCtx(ctx)
+
+			edges, err := youtube.ParseEdges(follow)
+			if err != nil {
+				return usageErr(err.Error())
+			}
+			seeds, err := parseSeeds(args)
+			if err != nil {
+				return err
+			}
+
+			var st *youtube.Store
+			if store {
+				st, err = app.RequireStore()
+				if err != nil {
+					return err
+				}
+			}
+
+			budget := app.Limit
+			if budget <= 0 {
+				budget = defaultDiscoverBudget
+			}
+
+			opts := youtube.WalkOptions{
+				Depth:  depth,
+				Max:    budget,
+				Fanout: fanout,
+				Edges:  edges,
+				Note:   func(s string) { app.logf("note: %s", s) },
+			}
+			if st != nil {
+				opts.OnEdge = func(src, dst string, e youtube.Edge) {
+					_ = st.UpsertEdge(src, dst, string(e))
+				}
+			}
+
+			n := 0
+			err = app.Client.Walk(ctx, seeds, opts, func(nd *youtube.Node) error {
+				if st != nil {
+					_ = st.UpsertNode(nd)
+				}
+				if e := app.Out.Emit(nodeRow(nd)); e != nil {
+					return e
+				}
+				n++
+				return nil
+			})
+			if flushErr := app.Out.Flush(); flushErr != nil && err == nil {
+				err = flushErr
+			}
+			if err != nil {
+				// The only fatal walk error is a seed that could not be fetched;
+				// it surfaces like a failed single read. Deeper failures (a gated
+				// comment edge, a missing community tab) are notes, not errors.
+				return err
+			}
+			if n == 0 {
+				return noResults("nothing discovered")
+			}
+			return nil
+		},
+	}
+}
+
+// parseSeeds turns the positional arguments into walk seeds, reporting an
+// unrecognized reference as a usage error rather than a plain failure.
+func parseSeeds(args []string) ([]youtube.Seed, error) {
+	seeds := make([]youtube.Seed, 0, len(args))
+	for _, a := range args {
+		s, err := youtube.ParseSeed(a)
+		if err != nil {
+			return nil, usageErr(err.Error())
+		}
+		seeds = append(seeds, s)
+	}
+	return seeds, nil
+}

cli/rows.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"strconv"
+	"strings"
 
 	"github.com/tamnd/ytb-cli/youtube"
 )
@@ -10,6 +11,65 @@ func itoa(n int) string   { return strconv.Itoa(n) }
 func i64a(n int64) string { return strconv.FormatInt(n, 10) }
 func boola(b bool) string { return strconv.FormatBool(b) }
 
+// oneline flattens a value to a single short line for a curated column, so a
+// multi-line title or comment body never breaks the list and table layouts.
+func oneline(s string) string {
+	s = strings.Join(strings.Fields(s), " ")
+	if len(s) > 80 {
+		return s[:79] + "..."
+	}
+	return s
+}
+
+// nodeRow renders a graph node discovered by `ytb discover`. The curated columns
+// read the walk at a glance (how deep, by which edge, and a one-line who/what)
+// while the full typed node (with the nested video, channel, playlist, comment,
+// or post) rides in Value for json, jsonl, and templates.
+func nodeRow(n *youtube.Node) Row {
+	var id, who, summary, url string
+	switch n.Kind {
+	case youtube.KindVideo:
+		v := n.Video
+		id = v.VideoID
+		who = v.ChannelName
+		summary = oneline(v.Title)
+		url = v.URL
+		if url == "" {
+			url = "https://www.youtube.com/watch?v=" + v.VideoID
+		}
+	case youtube.KindChannel:
+		c := n.Channel
+		id = c.ChannelID
+		who = c.Handle
+		if who == "" {
+			who = c.Title
+		}
+		summary = oneline(c.Title)
+		url = c.URL
+	case youtube.KindPlaylist:
+		p := n.Playlist
+		id = p.PlaylistID
+		who = p.ChannelName
+		summary = oneline(p.Title)
+		url = p.URL
+	case youtube.KindComment:
+		c := n.Comment
+		id = c.ID
+		who = c.AuthorDisplayName
+		summary = oneline(c.TextDisplay)
+	case youtube.KindPost:
+		p := n.Post
+		id = p.PostID
+		who = p.AuthorName
+		summary = oneline(p.ContentText)
+	}
+	return Row{
+		Cols:  []string{"depth", "via", "kind", "id", "who", "summary", "url"},
+		Vals:  []string{itoa(n.Depth), string(n.Via), string(n.Kind), id, who, summary, url},
+		Value: n,
+	}
+}
+
 func videoRow(v youtube.Video) Row {
 	return Row{
 		Cols: []string{"id", "title", "channel", "duration", "views", "published", "url"},

docs/content/guides/graph-discovery.md

@@ -0,0 +1,125 @@
+---
+title: "Graph discovery"
+description: "Walk the graph linked from a video, channel, or playlist breadth-first with ytb discover: follow uploaders, related videos, uploads, playlists, items, owners, community posts, comments, and the channels behind them."
+weight: 55
+---
+
+Every other read answers one question about one object: a video's metadata, a
+channel's uploads, a playlist's items. `ytb discover` chains them. It starts at a
+video, channel, or playlist and follows that object's links outward, hop by hop,
+streaming every node it reaches. It is a breadth-first walk of the YouTube graph.
+
+```sh
+ytb discover dQw4w9WgXcQ      # what is this video linked to?
+ytb discover @MrBeast         # what is this channel linked to?
+```
+
+A *seed* is any reference ytb already resolves: a video id or watch URL, a
+channel id, `@handle`, or URL, or a playlist id or URL. Pass more than one to
+start the walk from several places at once.
+
+## What gets followed
+
+The default follows an object's **content**: a video yields its uploader and
+related videos, a channel yields its uploads, a playlist yields its items and
+owner. It spans every seed kind, so `ytb discover <anything>` does the obvious
+thing with no flags, and it stays on the open surface, so it needs nothing
+special.
+
+Choose what to follow with `--follow`. It takes a preset:
+
+```sh
+ytb discover <ref> --follow content    # uploader, related, uploads, items, owner (default)
+ytb discover <channel> --follow feed   # uploads, playlists, community posts
+ytb discover <video> --follow comments # comments and the channels that wrote them
+ytb discover <ref> --follow all        # every edge
+```
+
+or a comma-separated list of individual edges:
+
+```sh
+ytb discover <video> --follow channel,related
+ytb discover <channel> --follow uploads,playlists
+```
+
+The full edge vocabulary:
+
+| Edge | From → to | Gated | What it follows |
+|---|---|---|---|
+| `channel` | video → channel | no | the channel that uploaded the video |
+| `related` | video → video | no | a related / recommended video |
+| `comments` | video → comment | yes | a comment on the video |
+| `uploads` | channel → video | no | a video the channel uploaded |
+| `playlists` | channel → playlist | no | a playlist the channel owns |
+| `community` | channel → post | no | a community-tab post |
+| `items` | playlist → video | no | a video in the playlist |
+| `owner` | playlist → channel | no | the channel that owns the playlist |
+| `commenter` | comment → channel | yes | the channel that wrote the comment |
+
+Name an edge directly to chase just that link: `--follow related` walks the
+recommendation graph, `--follow uploads` sweeps a channel.
+
+YouTube serves the open surface to an anonymous request, so there are no tiers to
+add. The only gated edges are the two that touch comments: YouTube applies a
+per-IP Restricted Mode to some networks, and when it refuses the comments,
+`ytb discover` notes it on stderr and keeps going on the rest of the graph rather
+than failing the walk. So `--follow all` from a flagged network still returns the
+channel, related videos, playlists, and everything else, with one advisory about
+the comments it could not read.
+
+## How far and how wide
+
+```sh
+ytb discover <ref> --depth 2          # follow two hops from the seed (default 1)
+ytb discover <ref> --fanout 50        # up to 50 neighbors per edge (default 25)
+ytb discover <ref> --fanout 0         # no per-edge cap
+ytb discover <ref> -n 1000            # stop after 1000 nodes total (default 500)
+```
+
+`--depth` is how many hops to follow; `0` emits only the seeds. `--fanout` caps
+how many neighbors each edge contributes per node, so one hop never pages a whole
+upload history unless you raise it. `-n/--limit` is the total node budget, the
+hard stop on a deep or wide walk; even `--fanout 0` stays bounded by it, so a
+walk always terminates.
+
+## Reading the output
+
+`ytb discover` streams one row per node, tagged with how it was reached:
+
+```text
+depth  via      kind     id           who        summary                url
+0               video    dQw4w9WgXcQ  RickAstley  Never Gonna Give You U https://youtu.be/dQw4w9WgXcQ
+1      channel  channel  UCuAXFkgsw1L  RickAstley  Rick Astley           https://youtube.com/channel/UCuAXFkgsw1L
+1      related  video    J---aiyznGQ  ...         Keyboard Cat           https://youtu.be/J---aiyznGQ
+```
+
+Because it streams through the same formatter as every read, it shapes and pipes
+the same way. The JSON forms carry the full typed node, with the nested video,
+channel, playlist, comment, or post:
+
+```sh
+ytb discover <ref> -o json | jq -r '.via + " -> " + (.video.id // .channel.id)'
+ytb discover @MrBeast --follow feed -o jsonl | jq -r '.video.id' | sort -u
+ytb discover <ref> --fields depth,via,who,url -o table
+```
+
+## Persisting a walk
+
+Add `--store` to write every node and edge into the local store as the walk
+streams, so you keep the graph as well as see it:
+
+```sh
+ytb discover @MrBeast --follow all --depth 2 --store
+ytb db query "select kind, count(*) from edges group by kind order by 2 desc"
+```
+
+Each node lands in its own typed table (`videos`, `channels`, `playlists`,
+`comments`, `community_posts`) and each traversed link lands in an `edges` table
+(`src`, `dst`, `kind`), so the graph is queryable afterwards. Re-walking is
+idempotent.
+
+When you want a dataset built from an explicit worklist rather than a live walk,
+reach for [the crawl queue](/guides/the-store/), which drains a queue of URLs you
+load yourself; `discover` is the complement that finds the worklist by walking.
+See [the local store](/guides/the-store/) for inspecting and exporting what you
+collect.

docs/content/guides/the-store.md

@@ -19,6 +19,22 @@ The store is pure Go (modernc.org/sqlite), so nothing links libsqlite and the
 binary stays static. Without `--db`, no database is ever created: the flag is the
 only thing that turns persistence on.
 
+## Persist a walk
+
+`ytb discover --store` tees a breadth-first walk into the store as a side effect,
+so a live walk doubles as a crawl. Every node lands in its typed table and every
+traversed link lands in an `edges` table, so the graph is queryable afterwards:
+
+```sh
+ytb discover @MrBeast --follow all --depth 2 --store
+ytb db query "select kind, count(*) from edges group by kind order by 2 desc"
+```
+
+See [graph discovery](/guides/graph-discovery/) for the full edge and preset
+vocabulary. The difference from the crawl queue below is direction: `discover`
+finds the worklist by walking the graph, while `seed`/`crawl` drain a worklist
+you load yourself.
+
 ## Building a crawl queue
 
 For larger collection runs, `seed`, `crawl`, `queue`, and `jobs` turn the store