Add a flag to control task retry appending/prepending

README.md

@@ -60,6 +60,10 @@ Available flags:
 	- Specifies the number of retries for a scraping task
 	- Type: unsigned integer
 	- Default value: `3`
+- `-taskretryfront`
+	- Prepend failed tasks to the front of the queue. If false, failed tasks go to the back of the queue
+	- Type: boolean
+	- Default value: `false`
 - `-verbose`
 	- Allows to put the app into verbose mode and print out additional logs to stdout
 	- Default value: none, no additional output is produced

main.go

@@ -30,6 +30,7 @@ func main() {
 	flagScraperFailurePause := flag.Int("scraperfailurepause", -1, "Amount of time in seconds to wait after a failed task to idle")
 	flagScraperThrottle := flag.Uint("scraperthrottle", 20, "Maximum number of requests scraper sends per 30 seconds")
 	flagTaskRetries := flag.Uint("taskretries", 3, "Number of retries for a scraping task")
+	flagTaskRetryFront := flag.Bool("taskretryfront", false, "Prepend failed tasks to the front of the queue")
 	flagVerbose := flag.Bool("verbose", false, "Print out additional logs into stdout")
 	flag.Parse()
 
@@ -65,6 +66,7 @@ func main() {
 	viper.Set("scraperfailurepause", time.Duration(*flagScraperFailurePause)*time.Second)
 	viper.Set("scraperthrottle", int(*flagScraperThrottle))
 	viper.Set("taskretries", int(*flagTaskRetries))
+	viper.Set("taskretryfront", *flagTaskRetryFront)
 	viper.Set("verbose", *flagVerbose)
 
 	cache.InitCache()

scraper/handleTaskError.go

@@ -48,7 +48,7 @@ func handleTaskError(r *colly.Request, blocked bool, err error) {
 			taskClient,
 			taskHash,
 			r.URL.String(),
-			true,
+			viper.GetBool("taskretryfront"),
 			map[string]string{
 				metadataTaskAddedAt: r.Ctx.Get(metadataTaskAddedAt),
 				metadataTaskClient:  taskClient,