diff --git a/go.mod b/go.mod index c812808..22d9ebe 100644 --- a/go.mod +++ b/go.mod @@ -129,7 +129,11 @@ require ( golang.org/x/term v0.37.0 // indirect golang.org/x/text v0.31.0 // indirect google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect - google.golang.org/protobuf v1.36.10 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect pluginrpc.com/pluginrpc v0.5.0 // indirect ) + +// See https://github.com/PowerDNS/lightningstream/issues/112 +// Can be removed once https://github.com/CrowdStrike/csproto/pull/208 is merged and released +replace github.com/CrowdStrike/csproto => github.com/wojas/csproto v0.0.0-20260107092112-0e013c7984a2 diff --git a/go.sum b/go.sum index 902fadf..8b7c8f1 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,6 @@ dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/CrowdStrike/csproto v0.35.0 h1:9vEV5PwDHXqinbobsI1Doy6kF5uHJyo4znYqXaUKQNs= -github.com/CrowdStrike/csproto v0.35.0/go.mod h1:KVvLr7dlTgOmX5oT4cw9mZ0rhMIBG2LHb8yt9DMg7ic= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -322,6 +320,8 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +github.com/wojas/csproto v0.0.0-20260107092112-0e013c7984a2 h1:T9S4tRljv07jn5KNFyKhecgU6rbsq0OTxxar8ATxP/U= +github.com/wojas/csproto v0.0.0-20260107092112-0e013c7984a2/go.mod h1:s4KQGP1RDkm7AyniRi6BUAWySd8RtXZmk1ZBm/CF1F8= github.com/wojas/go-healthz v0.2.0 h1:Pm+V2mCkEMvLoppiOW6zV/VpnbOfnwIA7F8Cd+ucmyg= github.com/wojas/go-healthz v0.2.0/go.mod h1:uAGwtiPYhG63AgJoWM+EFzSfwgW3rj58ZalNogNFRRk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -473,8 +473,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 58d7b84..06b31e7 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -4,6 +4,7 @@ import ( "io" "github.com/CrowdStrike/csproto" + "github.com/c2h5oh/datasize" ) // Protobuf field numbers @@ -14,6 +15,12 @@ const ( FieldSnapshotCompatVersion = 4 ) +// MaxFieldLength overrides the default 2GB limit on variable length protobuf +// fields to allow larger dumps with our current schema. +const ( + MaxFieldLength = 100 * uint64(datasize.GB) +) + // Snapshot is the root object in a snapshot protobuf type Snapshot struct { FormatVersion uint32 // version of this snapshot format @@ -25,6 +32,7 @@ type Snapshot struct { func (s *Snapshot) Unmarshal(data []byte) error { d := csproto.NewDecoder(data) d.SetMode(csproto.DecoderModeFast) + d.SetMaxFieldLength(MaxFieldLength) // allow DBI dumps larger than 2GB for d.More() { tag, wireType, err := d.DecodeTag() if err != nil {