Skip to content

Fix potential task requeueing failure #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
man90es merged 3 commits into from
May 25, 2026
Merged

Fix potential task requeueing failure #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d58149f
Fix potential task requeueing fail
man90es May 25, 2026
1e60407
Move confirming task completion to a script
man90es May 25, 2026
76befb2
Bump version number
man90es May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion handlers/getStatus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
)

var initTime = time.Now()
var version = "1.19.0"
var version = "1.19.1"

func getStatus(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(map[string]interface{}{
Expand Down
37 changes: 18 additions & 19 deletions scraper/handleTaskError.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,24 @@ func handleTaskError(r *colly.Request, blocked bool, err error) {
taskClient := r.Ctx.Get(metadataTaskClient)
taskHash := r.Ctx.Get(metadataTaskHash)

taskQueue.ConfirmTaskCompletion(taskClient, taskHash)
if taskRetries < viper.GetInt("taskretries") {
taskQueue.AddTask(
taskClient,
taskHash,
r.URL.String(),
viper.GetBool("taskretryfront"),
map[string]string{
metadataTaskAddedAt: r.Ctx.Get(metadataTaskAddedAt),
metadataTaskClient: taskClient,
metadataTaskHash: taskHash,
metadataTaskRegion: r.Ctx.Get(metadataTaskRegion),
metadataTaskRetries: strconv.Itoa(taskRetries + 1),
metadataTaskType: r.Ctx.Get(metadataTaskType),
},
)
}

if blocked {
logger.Error(fmt.Sprintf("Hit Imperva while loading %v, retries: %v", r.URL, taskRetries))
} else if strings.Contains(err.Error(), "http2: Transport received GOAWAY from server ErrCode:INTERNAL_ERROR") {
Expand All @@ -40,23 +58,4 @@ func handleTaskError(r *colly.Request, blocked bool, err error) {
} else {
taskQueue.Pause(time.Duration(60-time.Now().Second()) * time.Second)
}

taskQueue.ConfirmTaskCompletion(taskClient, taskHash)

if taskRetries < viper.GetInt("taskretries") {
taskQueue.AddTask(
taskClient,
taskHash,
r.URL.String(),
viper.GetBool("taskretryfront"),
map[string]string{
metadataTaskAddedAt: r.Ctx.Get(metadataTaskAddedAt),
metadataTaskClient: taskClient,
metadataTaskHash: taskHash,
metadataTaskRegion: r.Ctx.Get(metadataTaskRegion),
metadataTaskRetries: strconv.Itoa(taskRetries + 1),
metadataTaskType: r.Ctx.Get(metadataTaskType),
},
)
}
}
14 changes: 10 additions & 4 deletions scraper/taskQueue.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ var enqueueScript = redis.NewScript(`
return 1
`)

var confirmCompletionScript = redis.NewScript(`
local clientsKey, hashesKey = KEYS[1], KEYS[2]
local taskClient, hash = ARGV[1], ARGV[2]

redis.call("HINCRBY", clientsKey, taskClient, -1)
redis.call("SREM", hashesKey, hash)
return 1
`)

type Task struct {
Hash string
Metadata map[string]string
Expand Down Expand Up @@ -198,8 +207,5 @@ func (q *TaskQueue) CountQueuedTasksForClient(taskClient string) (count int) {
}

func (q *TaskQueue) ConfirmTaskCompletion(taskClient string, hash string) {
pipe := q.rdb.Pipeline()
pipe.HIncrBy(q.ctx, q.clientsKey, taskClient, -1)
pipe.SRem(q.ctx, q.hashesKey, hash)
_, _ = pipe.Exec(q.ctx)
_, _ = confirmCompletionScript.Run(q.ctx, q.rdb, []string{q.clientsKey, q.hashesKey}, taskClient, hash).Result()
}