wip: support paging when fetching list of words

3 files changed, 53 insertions, 23 deletions

diff --git a/client.go b/client.go
index ff30e73..d9781ab 100644
--- a/client.go
+++ b/client.go
@@ -149,18 +149,30 @@ func (cl *Client) Lookup(ins []string) (res LookupResponse, err error) {
 }
 
 // ListRootWords list all of the root words in dictionary.
-func (cl *Client) ListRootWords() (rootWords Words, err error) {
-	params := url.Values{
-		paramNameMasukan:        []string{paramValueDasar},
-		paramNameMasukanLengkap: []string{paramValueDasar},
+func (cl *Client) ListRootWords(pageStart, pageEnd int) (rootWords Words, err error) {
+	if pageStart < 1 {
+		pageStart = 1
+	}
+	if pageEnd < 1 {
+		pageEnd = maxPageNumber
 	}
 
+	var (
+		params = url.Values{
+			paramNameMasukan:        []string{paramValueDasar},
+			paramNameMasukanLengkap: []string{paramValueDasar},
+		}
+
+		got Words
+		n   int
+	)
+
 	urlPage := kbbiUrlBase + "/Cari/Jenis?"
 
 	rootWords = make(Words)
 
-	for pageNumber := 1; pageNumber <= maxPageNumber; pageNumber++ {
-		params.Set(paramNamePage, strconv.Itoa(pageNumber))
+	for ; pageStart <= pageEnd; pageStart++ {
+		params.Set(paramNamePage, strconv.Itoa(pageStart))
 
 		req, err := http.NewRequest(http.MethodGet, urlPage+params.Encode(), nil)
 		if err != nil {
@@ -169,31 +181,34 @@ func (cl *Client) ListRootWords() (rootWords Words, err error) {
 
 		res, err := cl.httpc.Do(req)
 		if err != nil {
-			return rootWords, fmt.Errorf("ListRootWords: page %d: %w",
-				pageNumber, err)
+			return rootWords, fmt.Errorf("ListRootWords: page %d: %w", pageStart, err)
 		}
 
 		defer res.Body.Close()
 
 		body, err := io.ReadAll(res.Body)
 		if err != nil {
-			return rootWords, fmt.Errorf("ListRootWords: page %d: %w",
-				pageNumber, err)
+			return rootWords, fmt.Errorf("ListRootWords: page %d: %w", pageStart, err)
 		}
 
-		got, err := cl.parseHTMLRootWords(body)
+		got, err = cl.parseHTMLRootWords(body)
 		if err != nil {
-			return rootWords, fmt.Errorf("ListRootWords: page %d: %w",
-				pageNumber, err)
+			return rootWords, fmt.Errorf("ListRootWords: page %d: %w", pageStart, err)
 		}
 		if len(got) == 0 {
 			break
 		}
 
-		rootWords.merge(got)
+		fmt.Printf("%d: got words: %v\n", pageStart, got)
+
+		n = rootWords.merge(got)
+		if n == 0 {
+			// The page does not have new words.
+			break
+		}
 
 		log.Printf("ListRootWords: halaman %d, jumlah kata %d, total kata %d",
-			pageNumber, len(got), len(rootWords))
+			pageStart, len(got), len(rootWords))
 	}
 
 	return rootWords, nil
diff --git a/cmd/kbbi/main.go b/cmd/kbbi/main.go
index 7db6aa6..044b2c7 100644
--- a/cmd/kbbi/main.go
+++ b/cmd/kbbi/main.go
@@ -19,13 +19,17 @@ const (
 	cmdNameSurel           = "surel"
 	cmdNameSandi           = "sandi"
 	cmdNameDaftarKataDasar = "daftar-kata-dasar"
+	optPageStart           = `page-start`
+	optPageEnd             = `page-end`
 )
 
 func main() {
 	var (
-		isListRootWords bool
 		email           string
 		pass            string
+		pageStart       int
+		pageEnd         int
+		isListRootWords bool
 	)
 
 	log.SetFlags(0)
@@ -33,6 +37,8 @@ func main() {
 
 	flag.StringVar(&email, cmdNameSurel, "", "Nama pengguna")
 	flag.StringVar(&pass, cmdNameSandi, "", "Sandi pengguna")
+	flag.IntVar(&pageStart, optPageStart, 1, `Mulai ambil kata dasar dari halaman ini`)
+	flag.IntVar(&pageEnd, optPageEnd, 0, `Berhenti ambil kata dasar pada halaman ini`)
 	flag.BoolVar(&isListRootWords, cmdNameDaftarKataDasar, false,
 		"Ambil dan cetak semua kata dasar")
 
@@ -56,7 +62,7 @@ func main() {
 				cmdNameDaftarKataDasar, cmdNameSurel,
 				cmdNameSandi)
 		}
-		listRootWords(cl)
+		listRootWords(cl, pageStart, pageEnd)
 		return
 	}
 
@@ -68,8 +74,8 @@ func main() {
 	resDefinition.Write(os.Stdout)
 }
 
-func listRootWords(cl *kbbi.Client) {
-	words, err := cl.ListRootWords()
+func listRootWords(cl *kbbi.Client, pageStart, pageEnd int) {
+	words, err := cl.ListRootWords(pageStart, pageEnd)
 	if err != nil {
 		log.Println(err)
 	}
diff --git a/words.go b/words.go
index e5af469..904f644 100644
--- a/words.go
+++ b/words.go
@@ -7,9 +7,18 @@ package kbbi
 type Words map[string]struct{}
 
 // merge other map into current map.
-func (words Words) merge(in Words) Words {
-	for k := range in {
-		words[k] = struct{}{}
+func (words Words) merge(in Words) (n int) {
+	var (
+		word  string
+		exist bool
+	)
+	for word = range in {
+		_, exist = words[word]
+		if exist {
+			continue
+		}
+		words[word] = struct{}{}
+		n++
 	}
-	return words
+	return n
 }