diff --git a/README.md b/README.md index 7d0ba47..46d9fc4 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,10 @@ Available flags: - Specifies the number of retries for a scraping task - Type: unsigned integer - Default value: `3` +- `-taskretryfront` + - Prepend failed tasks to the front of the queue. If false, failed tasks go to the back of the queue + - Type: boolean + - Default value: `false` - `-verbose` - Allows to put the app into verbose mode and print out additional logs to stdout - Default value: none, no additional output is produced diff --git a/main.go b/main.go index 4ad706a..6cbc318 100644 --- a/main.go +++ b/main.go @@ -30,6 +30,7 @@ func main() { flagScraperFailurePause := flag.Int("scraperfailurepause", -1, "Amount of time in seconds to wait after a failed task to idle") flagScraperThrottle := flag.Uint("scraperthrottle", 20, "Maximum number of requests scraper sends per 30 seconds") flagTaskRetries := flag.Uint("taskretries", 3, "Number of retries for a scraping task") + flagTaskRetryFront := flag.Bool("taskretryfront", false, "Prepend failed tasks to the front of the queue") flagVerbose := flag.Bool("verbose", false, "Print out additional logs into stdout") flag.Parse() @@ -65,6 +66,7 @@ func main() { viper.Set("scraperfailurepause", time.Duration(*flagScraperFailurePause)*time.Second) viper.Set("scraperthrottle", int(*flagScraperThrottle)) viper.Set("taskretries", int(*flagTaskRetries)) + viper.Set("taskretryfront", *flagTaskRetryFront) viper.Set("verbose", *flagVerbose) cache.InitCache() diff --git a/scraper/handleTaskError.go b/scraper/handleTaskError.go index 3fefd82..f35278e 100644 --- a/scraper/handleTaskError.go +++ b/scraper/handleTaskError.go @@ -48,7 +48,7 @@ func handleTaskError(r *colly.Request, blocked bool, err error) { taskClient, taskHash, r.URL.String(), - true, + viper.GetBool("taskretryfront"), map[string]string{ metadataTaskAddedAt: r.Ctx.Get(metadataTaskAddedAt), metadataTaskClient: taskClient,