Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions sdks/go/pkg/beam/runners/dataflow/dataflowlib/execute.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ func Execute(ctx context.Context, raw *pipepb.Pipeline, opts *JobOptions, worker
if err != nil {
return presult, err
}
opts.WorkerHash = hash
log.Infof(ctx, "Staged worker binary: %v", workerURL)

if err := graphx.UpdateDefaultEnvWorkerType(
Expand Down
4 changes: 4 additions & 0 deletions sdks/go/pkg/beam/runners/dataflow/dataflowlib/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ type JobOptions struct {
// Worker is the worker binary override.
Worker string

// WorkerHash is the SHA-256 hash of the worker binary.
WorkerHash string

// -- Internal use only. Not supported in public Dataflow. --

TeardownPolicy string
Expand Down Expand Up @@ -136,6 +139,7 @@ func Translate(ctx context.Context, p *pipepb.Pipeline, opts *JobOptions, worker
packages := []*df.Package{{
Name: "worker",
Location: workerURL,
Sha256: opts.WorkerHash,
}}

for _, url := range opts.ArtifactURLs {
Expand Down
16 changes: 16 additions & 0 deletions sdks/go/pkg/beam/runners/dataflow/dataflowlib/job_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ func TestTranslate(t *testing.T) {
Name: "test-job",
DiskProvisionedIops: 4000,
DiskProvisionedThroughputMibps: 200,
WorkerHash: "worker-sha256-hash",
}
workerURL := "gs://any-location/temp"
modelURL := "gs://any-location/temp"
Expand All @@ -312,6 +313,21 @@ func TestTranslate(t *testing.T) {
if wp.DiskProvisionedThroughputMibps != 200 {
t.Errorf("DiskProvisionedThroughputMibps = %v, want 200", wp.DiskProvisionedThroughputMibps)
}

// Verify worker package has Sha256
found := false
for _, pkg := range wp.Packages {
if pkg.Name == "worker" {
found = true
if pkg.Sha256 != "worker-sha256-hash" {
t.Errorf("worker package Sha256 = %q, want %q", pkg.Sha256, "worker-sha256-hash")
}
break
}
}
if !found {
t.Fatalf("worker package not found in wp.Packages")
}
Comment thread
tarun-google marked this conversation as resolved.
}

func TestTranslateWithPipelineHash(t *testing.T) {
Expand Down
Loading