all: add test cases for simulating slow server

The test run a server that contains three six pages that contains various [time.Sleep] duration before returning the response. This allow us to see how the main scan loop works, waiting for resultq and listWaitStatus.
author: Shulhan <ms@kilabit.info> 2025-06-19 01:06:48 +0700
committer: Shulhan <ms@kilabit.info> 2025-06-19 01:06:48 +0700
commit: 8bc8fce1bd80b5a25c452ac5a24b1a1e3f5a4feb (patch)
tree: 8fa07a3fb043655aae542c9fad71098c3f2a6d84
parent: 5ac82928702a0f0a7be0ef6e96ab04c39a7e8e9d (diff)
download: jarink-8bc8fce1bd80b5a25c452ac5a24b1a1e3f5a4feb.tar.xz
2 files changed, 143 insertions, 42 deletions
diff --git a/brokenlinks/brokenlinks_test.go b/brokenlinks/brokenlinks_test.go
index 6feb1e2..d9f9a59 100644
--- a/brokenlinks/brokenlinks_test.go
+++ b/brokenlinks/brokenlinks_test.go
@@ -4,6 +4,7 @@
 package brokenlinks_test
 
 import (
+	"encoding/json"
 	"log"
 	"net/http"
 	"os"
@@ -16,21 +17,27 @@ import (
 	"git.sr.ht/~shulhan/jarink/brokenlinks"
 )
 
-// The test run three web servers that serve content on "testdata/web/".
-// The first web server is the one that we want to scan.
-// The second web server is external web server, where HTML pages should not
+// The test run four web servers.
+//
+// The first web server is the one that we want to scan, it serve content on
+// "testdata/web".
+//
+// The second web server is an external web server, where HTML pages should not
 // be parsed.
+//
 // The third web server is with insecure, self-signed TLS, for testing
 // "insecure" option.
-//
-// Command to generate certificate:
+// Commands to generate certificate:
 //	$ openssl genrsa -out 127.0.0.1.key
 //	$ openssl x509 -new -key=127.0.0.1.key -subj="/CN=shulhan" \
 //		-days=3650 -out=127.0.0.1.pem
+//
+// The fourth web server is slow response web server.
 
 const testAddress = `127.0.0.1:11836`
 const testExternalAddress = `127.0.0.1:11900`
 const testInsecureAddress = `127.0.0.1:11838`
+const testAddressSlow = `127.0.0.1:11839`
 
 func TestMain(m *testing.M) {
 	log.SetFlags(0)
@@ -40,6 +47,7 @@ func TestMain(m *testing.M) {
 	go testServer(fshandle)
 	go testExternalServer(fshandle)
 	go testInsecureServer(fshandle)
+	go runServerSlow(testAddressSlow)
 
 	var err = libnet.WaitAlive(`tcp`, testAddress, 5*time.Second)
 	if err != nil {
@@ -53,6 +61,10 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.Fatal(err)
 	}
+	err = libnet.WaitAlive(`tcp`, testAddressSlow, 5*time.Second)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	os.Exit(m.Run())
 }
@@ -112,6 +124,80 @@ func testInsecureServer(fshandle http.Handler) {
 	}
 }
 
+func runServerSlow(addr string) {
+	var mux = http.NewServeMux()
+	mux.HandleFunc(`/`, func(resp http.ResponseWriter, req *http.Request) {
+		resp.WriteHeader(http.StatusOK)
+		var body = []byte(`<html><body>
+			<a href="/slow1">Slow 1</a>
+			<a href="/slow2">Slow 2</a>
+			<a href="/slow3">Slow 3</a>
+			</body></html>`)
+		resp.Write(body)
+	})
+
+	mux.HandleFunc(`/slow1`,
+		func(resp http.ResponseWriter, req *http.Request) {
+			resp.WriteHeader(http.StatusOK)
+			var body = []byte(`<html><body>
+				<a href="/slow1/sub">Slow 1, sub</a>
+				<a href="/slow2/sub">Slow 2, sub</a>
+				<a href="/slow3/sub">Slow 3, sub</a>
+				</body></html>`)
+			resp.Write(body)
+		})
+	mux.HandleFunc(`/slow2`,
+		func(resp http.ResponseWriter, req *http.Request) {
+			time.Sleep(1 * time.Second)
+			resp.WriteHeader(http.StatusOK)
+			var body = []byte(`<html><body>
+				<a href="/slow1/sub">Slow 1, sub</a>
+				<a href="/slow2/sub">Slow 2, sub</a>
+				<a href="/slow3/sub">Slow 3, sub</a>
+				</body></html>`)
+			resp.Write(body)
+		})
+	mux.HandleFunc(`/slow3`,
+		func(resp http.ResponseWriter, req *http.Request) {
+			time.Sleep(2 * time.Second)
+			resp.WriteHeader(http.StatusOK)
+			var body = []byte(`<html><body>
+				<a href="/slow1/sub">Slow 1, sub</a>
+				<a href="/slow2/sub">Slow 2, sub</a>
+				<a href="/slow3/sub">Slow 3, sub</a>
+				</body></html>`)
+			resp.Write(body)
+		})
+
+	mux.HandleFunc(`/slow1/sub`,
+		func(resp http.ResponseWriter, req *http.Request) {
+			time.Sleep(4 * time.Second)
+			resp.WriteHeader(http.StatusOK)
+		})
+	mux.HandleFunc(`/slow2/sub`,
+		func(resp http.ResponseWriter, req *http.Request) {
+			time.Sleep(5 * time.Second)
+			resp.WriteHeader(http.StatusOK)
+		})
+	mux.HandleFunc(`/slow3/sub`,
+		func(resp http.ResponseWriter, req *http.Request) {
+			time.Sleep(6 * time.Second)
+			resp.WriteHeader(http.StatusForbidden)
+		})
+
+	var httpServer = &http.Server{
+		Addr:           addr,
+		Handler:        mux,
+		ReadTimeout:    10 * time.Second,
+		WriteTimeout:   10 * time.Second,
+		MaxHeaderBytes: 1 << 20,
+	}
+	var err = httpServer.ListenAndServe()
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
 func TestScan(t *testing.T) {
 	var testUrl = `http://` + testAddress
 
@@ -284,3 +370,39 @@ func TestScan_pastResult(t *testing.T) {
 		test.Assert(t, tcase.opts.Url, tcase.exp, result.BrokenLinks)
 	}
 }
+
+func TestScan_slow(t *testing.T) {
+	const testUrl = `http://` + testAddressSlow
+
+	var opts = brokenlinks.Options{
+		Url: testUrl,
+	}
+
+	var gotResult *brokenlinks.Result
+	var err error
+	gotResult, err = brokenlinks.Scan(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got, _ := json.MarshalIndent(gotResult, ``, `  `)
+	t.Logf(`got=%s`, got)
+
+	var expResult = &brokenlinks.Result{
+		BrokenLinks: map[string][]brokenlinks.Broken{
+			testUrl + `/slow1`: []brokenlinks.Broken{{
+				Link: testUrl + `/slow3/sub`,
+				Code: http.StatusForbidden,
+			}},
+			testUrl + `/slow2`: []brokenlinks.Broken{{
+				Link: testUrl + `/slow3/sub`,
+				Code: http.StatusForbidden,
+			}},
+			testUrl + `/slow3`: []brokenlinks.Broken{{
+				Link: testUrl + `/slow3/sub`,
+				Code: http.StatusForbidden,
+			}},
+		},
+	}
+	test.Assert(t, `TestScan_slow`, expResult, gotResult)
+}
diff --git a/brokenlinks/worker.go b/brokenlinks/worker.go
index 94be90b..3eacf01 100644
--- a/brokenlinks/worker.go
+++ b/brokenlinks/worker.go
@@ -145,49 +145,29 @@ func (wrk *worker) scanAll() (result *Result, err error) {
 		go wrk.scan(linkq)
 	}
 
-	var tick = time.NewTicker(500 * time.Millisecond)
-	var listWaitStatus []linkQueue
-	var isScanning = true
-	for isScanning {
-		select {
-		case resultq := <-wrk.resultq:
-			listWaitStatus = wrk.processResult(resultq, listWaitStatus)
-
-		case <-tick.C:
-			wrk.wg.Wait()
-			if len(wrk.resultq) != 0 {
-				continue
-			}
-			if len(listWaitStatus) != 0 {
-				// There are links that still waiting for
-				// scanning to be completed.
-				continue
-			}
-			isScanning = false
-		}
-	}
-	wrk.result.sort()
+	wrk.processAndWait()
 	return wrk.result, nil
 }
 
 // scanPastResult scan only pages reported inside
 // [Result.BrokenLinks].
-func (wrk *worker) scanPastResult() (
-	result *Result, err error,
-) {
-	go func() {
-		for page := range wrk.pastResult.BrokenLinks {
-			var linkq = linkQueue{
-				parentUrl: nil,
-				url:       page,
-				status:    http.StatusProcessing,
-			}
-			wrk.seenLink[linkq.url] = http.StatusProcessing
-			wrk.wg.Add(1)
-			go wrk.scan(linkq)
+func (wrk *worker) scanPastResult() (result *Result, err error) {
+	for page := range wrk.pastResult.BrokenLinks {
+		var linkq = linkQueue{
+			parentUrl: nil,
+			url:       page,
+			status:    http.StatusProcessing,
 		}
-	}()
+		wrk.seenLink[linkq.url] = http.StatusProcessing
+		wrk.wg.Add(1)
+		go wrk.scan(linkq)
+	}
 
+	wrk.processAndWait()
+	return wrk.result, nil
+}
+
+func (wrk *worker) processAndWait() {
 	var tick = time.NewTicker(500 * time.Millisecond)
 	var listWaitStatus []linkQueue
 	var isScanning = true
@@ -210,7 +190,6 @@ func (wrk *worker) scanPastResult() (
 		}
 	}
 	wrk.result.sort()
-	return wrk.result, nil
 }
 
 // processResult the resultq contains the original URL being scanned
author	Shulhan <ms@kilabit.info>	2025-06-19 01:06:48 +0700
committer	Shulhan <ms@kilabit.info>	2025-06-19 01:06:48 +0700
commit	8bc8fce1bd80b5a25c452ac5a24b1a1e3f5a4feb (patch)
tree	8fa07a3fb043655aae542c9fad71098c3f2a6d84
parent	5ac82928702a0f0a7be0ef6e96ab04c39a7e8e9d (diff)
download	jarink-8bc8fce1bd80b5a25c452ac5a24b1a1e3f5a4feb.tar.xz