brokenlinks: add option "insecure"

The insecure option will allow and not report as error on server with
invalid certificates.

10 files changed, 121 insertions, 3 deletions

diff --git a/README.adoc b/README.adoc
index f85fa45..b8f79b0 100644
--- a/README.adoc
+++ b/README.adoc
@@ -49,6 +49,9 @@ This command accept the following options,
 `-ignore-status=<comma separated HTTP status code>`::
 List of HTTP status code that will be ignored during scan.
 
+`-insecure`::
+Do not report as error on server with invalid certificates.
+
 `-past-result=<path to JSON file>`::
 Scan only the pages reported by result from past scan based
 on the content in JSON file.
diff --git a/brokenlinks/brokenlinks_test.go b/brokenlinks/brokenlinks_test.go
index b868942..9b176b7 100644
--- a/brokenlinks/brokenlinks_test.go
+++ b/brokenlinks/brokenlinks_test.go
@@ -16,13 +16,21 @@ import (
 	"git.sr.ht/~shulhan/jarink/brokenlinks"
 )
 
-// The test run two web servers that serve content on "testdata/web/".
+// The test run three web servers that serve content on "testdata/web/".
 // The first web server is the one that we want to scan.
 // The second web server is external web server, where HTML pages should not
 // be parsed.
+// The third web server is with insecure, self-signed TLS, for testing
+// "insecure" option.
+//
+// Command to generate certificate:
+//	$ openssl genrsa -out 127.0.0.1.key
+//	$ openssl x509 -new -key=127.0.0.1.key -subj="/CN=shulhan" \
+//		-days=3650 -out=127.0.0.1.pem
 
 const testAddress = `127.0.0.1:11836`
 const testExternalAddress = `127.0.0.1:11900`
+const testInsecureAddress = `127.0.0.1:11838`
 
 func TestMain(m *testing.M) {
 	log.SetFlags(0)
@@ -31,6 +39,7 @@ func TestMain(m *testing.M) {
 
 	go testServer(fshandle)
 	go testExternalServer(fshandle)
+	go testInsecureServer(fshandle)
 
 	var err = libnet.WaitAlive(`tcp`, testAddress, 5*time.Second)
 	if err != nil {
@@ -40,6 +49,10 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		log.Fatal(err)
 	}
+	err = libnet.WaitAlive(`tcp`, testInsecureAddress, 5*time.Second)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	os.Exit(m.Run())
 }
@@ -81,6 +94,24 @@ func testExternalServer(fshandle http.Handler) {
 	}
 }
 
+func testInsecureServer(fshandle http.Handler) {
+	var mux = http.NewServeMux()
+	mux.Handle(`/`, fshandle)
+	var testServer = &http.Server{
+		Addr:           testInsecureAddress,
+		Handler:        mux,
+		ReadTimeout:    10 * time.Second,
+		WriteTimeout:   10 * time.Second,
+		MaxHeaderBytes: 1 << 20,
+	}
+	var certFile = `testdata/127.0.0.1.pem`
+	var keyFile = `testdata/127.0.0.1.key`
+	var err = testServer.ListenAndServeTLS(certFile, keyFile)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
 func TestBrokenlinks(t *testing.T) {
 	var testUrl = `http://` + testAddress
 
@@ -104,6 +135,7 @@ func TestBrokenlinks(t *testing.T) {
 		opts: brokenlinks.Options{
 			Url:          testUrl,
 			IgnoreStatus: `403`,
+			Insecure:     true,
 		},
 		exp: map[string][]brokenlinks.Broken{
 			testUrl: []brokenlinks.Broken{
diff --git a/brokenlinks/options.go b/brokenlinks/options.go
index 9c4022b..5a73b19 100644
--- a/brokenlinks/options.go
+++ b/brokenlinks/options.go
@@ -24,6 +24,9 @@ type Options struct {
 	ignoreStatus []int
 
 	IsVerbose bool
+
+	// Insecure do not report error on server with invalid certificates.
+	Insecure bool
 }
 
 func (opts *Options) init() (err error) {
diff --git a/brokenlinks/testdata/127.0.0.1.key b/brokenlinks/testdata/127.0.0.1.key
new file mode 100644
index 0000000..551c501
--- /dev/null
+++ b/brokenlinks/testdata/127.0.0.1.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCyImnwyRsQv7iA
+NePtvgdEWzoALE3FVernkHiXbOn50jf4vA3u0ENOW1bcoHT6Mq8hN7Psow/knXCD
+FWnPahAxLTNKiSQcC7Nt23YxTyvOEqltB63XjRB07aLRUSgp7Kr4rHr8QeMa3h0A
+hjn7EVQ7V2uOEjpYKnvAOJwd9uNq6Io9uiCR/HsAOa5c+tTN6tMiWFPaG4FET56t
++7rUxvtn1LCsEaJ+Bsh8OoMyd02lXoyzfnuw1Ip6NK5bL7zcJy7a5qi0DnizGtZt
+YcCQVmw7Gzf8ATYqUUBxDFUkPMnYPH1lWsoi02r2+ckpbEkgsixMYyKOABeTPvt3
+rO3zQnRJAgMBAAECggEACm0EtdTTanTKK+jbjQni+396YtG1AJCBhemZAUXc4OQK
+php9l0VltxOV/yY+CnnGccOEmpxFnRwQ3n81mO/sgDBU0O87Z4/QweSHI6YuWmqR
+s4GwTU4ikTFqZuVg3ClPrWvM6/BARk2LNYFlqdnvC+UXNLLA6vGbmhnGG2PcliBq
+al8nu7hXH3efdUsk0dBdSzFIDU+nL/9HIHpS1ZSjsxDpeXsa+Yr1GnWDqroaq9GJ
+OCK2wbkYM0oeMppq1E5Smt0lAj5fnK0GoRL0Unkjr1gmTuChjcxrBxhgooWH21Na
+9S5G7e7jU2T/vPzCFJ8T9J3LV5dyhszwmttprrghPQKBgQDyU1NsakRLftZlmLhN
+YXp2ou1AICtGCp452RumV2ZwU9cF2kmz5WUCZ6lUsvd9A2YMfwiLgwU9IAOT5Z8n
+05ZjPL/rdNSPbpA0HCRxSByEfQ0VJDlmAQNgVDOSlrujDOJhgyANS18aG0RTyj+r
+Ybd6gEyR41TjE9ZC/GqqeKtnLQKBgQC8L8XzwCIc/213lzhLIFeo/smdihJehWVN
+J6uVn1VHX28BtMKcx3R2v4PgC42JdYY9DXBKBBLQuNvTlSzo4Mcx4pxHH6pyBAtW
+6RocafuXp6Q1baaDoZ9gkhUx4fhkQVoOf/R8Y70Ltn75AG+z3KKGb6+WVJ7zRpCn
+iv3ioSlzDQKBgQCgyoWKZKDYb1sXotR4E/xMiCNg73eHTDhdLiBYqZnBYwBEU+mf
+wtDZXatQFkh93SlzlT84Q0HQo1N8aVrH9G/PfVnjhGwemEB7M0lDGZRCnS93Hcgw
+VO/GlVh7JiVvNXdpOLal26NJEVqvNn402+wBDuy/yNZkrp73Z8HnR5aEKQKBgQCq
+I7cg7bDp7rWVzg6DPbaDf/fgixiYhJpV62viVq/PW6UNMdRR0rKlOfmM8mUAxlSb
+li7TfGNWegule9WiprbxjyQj2alMdAOcjBujXN6u+k4oT/6gO9vQf8LR2q+sVLmL
+KnxsifA9Sr35ej+DqhL24Lsre05KPJ5EHBH2eCb1cQKBgHD8NUGfDTpityi7H8Wb
+K/qugwezTqtNTBk3rcZWtJT4J4covnn9ONSSEnomgUet/2V9PU1ZNPJF8ypP+rrS
+SqGasIPpi/WxVLehBJg381g6AL8zR9YC6Y+32dc166ya8lZ26FZk3/S+8xmZckN9
+AKHTtxukiJc/14g3TLTLt9Eg
+-----END PRIVATE KEY-----
diff --git a/brokenlinks/testdata/127.0.0.1.key.license b/brokenlinks/testdata/127.0.0.1.key.license
new file mode 100644
index 0000000..22616a9
--- /dev/null
+++ b/brokenlinks/testdata/127.0.0.1.key.license
@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2025 M. Shulhan <ms@kilabit.info>
+SPDX-License-Identifier: GPL-3.0-only
diff --git a/brokenlinks/testdata/127.0.0.1.pem b/brokenlinks/testdata/127.0.0.1.pem
new file mode 100644
index 0000000..da77616
--- /dev/null
+++ b/brokenlinks/testdata/127.0.0.1.pem
@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC0zCCAbugAwIBAgIUafednTys6Z4ZHfi+PllCYWm53IMwDQYJKoZIhvcNAQEL
+BQAwEjEQMA4GA1UEAwwHc2h1bGhhbjAeFw0yNTA2MTQwODQ2NDBaFw0zNTA2MTIw
+ODQ2NDBaMBIxEDAOBgNVBAMMB3NodWxoYW4wggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQCyImnwyRsQv7iANePtvgdEWzoALE3FVernkHiXbOn50jf4vA3u
+0ENOW1bcoHT6Mq8hN7Psow/knXCDFWnPahAxLTNKiSQcC7Nt23YxTyvOEqltB63X
+jRB07aLRUSgp7Kr4rHr8QeMa3h0Ahjn7EVQ7V2uOEjpYKnvAOJwd9uNq6Io9uiCR
+/HsAOa5c+tTN6tMiWFPaG4FET56t+7rUxvtn1LCsEaJ+Bsh8OoMyd02lXoyzfnuw
+1Ip6NK5bL7zcJy7a5qi0DnizGtZtYcCQVmw7Gzf8ATYqUUBxDFUkPMnYPH1lWsoi
+02r2+ckpbEkgsixMYyKOABeTPvt3rO3zQnRJAgMBAAGjITAfMB0GA1UdDgQWBBQg
+QwgV1gGRpltKcxyo2XawdmlgnjANBgkqhkiG9w0BAQsFAAOCAQEADlNnMyshG72E
+oYUtkdpV2pRyMbLmi4oaAGVGmIRoS6bj+hDe93gZZ93wH7nduoVYo/NlBrkuwjAb
+gEy0i/sAULm8ZAHIl8SViAbs5r+H4vBQrU3OsDBCmFQCwVuyr3gO6+EeiXq8/k6L
+e/rNTseHfslQxksP8wtpbkJ6lKf61na0rj/3mxzKu/QUXi73OLhq3gjfwvnzUTaP
+Yp9Z4tOFObsqJ1mEqoq6aOIec3YljnDSKL+/F9BQ6QQr+ncao4T9F1dcPnCO43yI
+nA3ztee5YZ1YwckUiAM5crcUzBoypXx0lcH6yF7kCyzhw0i201qJ46D5uWPAEN66
+JgK7bJpdFQ==
+-----END CERTIFICATE-----
diff --git a/brokenlinks/testdata/127.0.0.1.pem.license b/brokenlinks/testdata/127.0.0.1.pem.license
new file mode 100644
index 0000000..22616a9
--- /dev/null
+++ b/brokenlinks/testdata/127.0.0.1.pem.license
@@ -0,0 +1,2 @@
+SPDX-FileCopyrightText: 2025 M. Shulhan <ms@kilabit.info>
+SPDX-License-Identifier: GPL-3.0-only
diff --git a/brokenlinks/testdata/web/index.html b/brokenlinks/testdata/web/index.html
index cc11a2f..7b9101c 100644
--- a/brokenlinks/testdata/web/index.html
+++ b/brokenlinks/testdata/web/index.html
@@ -21,5 +21,8 @@ SPDX-License-Identifier: GPL-3.0-only
 
     <!-- Pages that return custom HTTP status code -->
     <a href="/page403">Page 403</a>
+
+    <!-- Pages with insecure TLS -->
+    <a href="https://127.0.0.1:11838">Insecure pages</a>
   </body>
 </html>
diff --git a/brokenlinks/worker.go b/brokenlinks/worker.go
index e3e0c45..8f278a8 100644
--- a/brokenlinks/worker.go
+++ b/brokenlinks/worker.go
@@ -4,6 +4,7 @@
 package brokenlinks
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -46,6 +47,8 @@ type worker struct {
 
 	log *log.Logger
 
+	httpc *http.Client
+
 	opts Options
 
 	// wg sync the goroutine scanner.
@@ -53,12 +56,31 @@ type worker struct {
 }
 
 func newWorker(opts Options) (wrk *worker, err error) {
+	var netDial = &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}
+	var tlsConfig = &tls.Config{
+		InsecureSkipVerify: opts.Insecure,
+	}
+
 	wrk = &worker{
 		opts:     opts,
 		seenLink: map[string]int{},
 		resultq:  make(chan map[string]linkQueue, 100),
 		result:   newResult(),
 		log:      log.New(os.Stderr, ``, log.LstdFlags),
+		httpc: &http.Client{
+			Transport: &http.Transport{
+				DialContext:           netDial.DialContext,
+				ExpectContinueTimeout: 1 * time.Second,
+				ForceAttemptHTTP2:     true,
+				IdleConnTimeout:       90 * time.Second,
+				MaxIdleConns:          100,
+				TLSClientConfig:       tlsConfig,
+				TLSHandshakeTimeout:   10 * time.Second,
+			},
+		},
 	}
 
 	wrk.scanUrl, err = url.Parse(opts.Url)
@@ -390,12 +412,12 @@ func (wrk *worker) fetch(linkq linkQueue) (
 			if wrk.opts.IsVerbose {
 				wrk.log.Printf("scan: HEAD %s\n", linkq.url)
 			}
-			httpResp, err = http.Head(linkq.url)
+			httpResp, err = wrk.httpc.Head(linkq.url)
 		} else {
 			if wrk.opts.IsVerbose {
 				wrk.log.Printf("scan: GET %s\n", linkq.url)
 			}
-			httpResp, err = http.Get(linkq.url)
+			httpResp, err = wrk.httpc.Get(linkq.url)
 		}
 		if err == nil {
 			return httpResp, nil
diff --git a/cmd/jarink/main.go b/cmd/jarink/main.go
index 9d6d1e8..88361ea 100644
--- a/cmd/jarink/main.go
+++ b/cmd/jarink/main.go
@@ -20,6 +20,7 @@ func main() {
 
 	var (
 		optIgnoreStatus string
+		optInsecure     bool
 		optIsVerbose    bool
 		optPastResult   string
 	)
@@ -27,6 +28,9 @@ func main() {
 	flag.StringVar(&optIgnoreStatus, `ignore-status`, ``,
 		`Comma separated HTTP response status code to be ignored.`)
 
+	flag.BoolVar(&optInsecure, `insecure`, false,
+		`Do not report as error on server with invalid certificates.`)
+
 	flag.BoolVar(&optIsVerbose, `verbose`, false,
 		`Print additional information while running.`)
 
@@ -41,6 +45,7 @@ func main() {
 	case `brokenlinks`:
 		var opts = brokenlinks.Options{
 			IgnoreStatus:   optIgnoreStatus,
+			Insecure:       optInsecure,
 			IsVerbose:      optIsVerbose,
 			PastResultFile: optPastResult,
 		}