all: check for DNS timeout and retry 5 times

When the call to HTTP HEAD or GET return an error and the error is
*net.DNSError with Timeout, retry the call until no error or Timeout
again for 5 times.

1 files changed, 35 insertions, 11 deletions

diff --git a/brokenlinks_worker.go b/brokenlinks_worker.go
index 03a89bd..14ad615 100644
--- a/brokenlinks_worker.go
+++ b/brokenlinks_worker.go
@@ -5,8 +5,10 @@ package jarink
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -296,17 +298,7 @@ func (wrk *brokenlinksWorker) scan(linkq linkQueue) {
 		httpResp *http.Response
 		err      error
 	)
-	if linkq.kind == atom.Img || linkq.isExternal {
-		if wrk.opts.IsVerbose {
-			wrk.log.Printf("scan: HEAD %s\n", linkq.url)
-		}
-		httpResp, err = http.Head(linkq.url)
-	} else {
-		if wrk.opts.IsVerbose {
-			wrk.log.Printf("scan: GET %s\n", linkq.url)
-		}
-		httpResp, err = http.Get(linkq.url)
-	}
+	httpResp, err = wrk.fetch(linkq)
 	if err != nil {
 		linkq.status = StatusBadLink
 		linkq.errScan = err
@@ -382,6 +374,38 @@ func (wrk *brokenlinksWorker) scan(linkq linkQueue) {
 	go wrk.pushResult(resultq)
 }
 
+func (wrk *brokenlinksWorker) fetch(linkq linkQueue) (
+	httpResp *http.Response,
+	err error,
+) {
+	const maxRetry = 5
+	var retry int
+	for retry < 5 {
+		if linkq.kind == atom.Img || linkq.isExternal {
+			if wrk.opts.IsVerbose {
+				wrk.log.Printf("scan: HEAD %s\n", linkq.url)
+			}
+			httpResp, err = http.Head(linkq.url)
+		} else {
+			if wrk.opts.IsVerbose {
+				wrk.log.Printf("scan: GET %s\n", linkq.url)
+			}
+			httpResp, err = http.Get(linkq.url)
+		}
+		if err == nil {
+			return httpResp, nil
+		}
+		var errDNS *net.DNSError
+		if !errors.As(err, &errDNS) {
+			return nil, err
+		}
+		if errDNS.Timeout() {
+			retry++
+		}
+	}
+	return nil, err
+}
+
 func (wrk *brokenlinksWorker) processLink(parentUrl *url.URL, val string, kind atom.Atom) (
 	linkq *linkQueue,
 ) {