all: add user interface to configure source of hosts block

The user interface, which can be accessed at
http://127.0.0.1:5380/#hostsblock, contains list of hosts block sources
that can be enabled or disabled using check box.

If enabled and the file is outdated, it will fetch the latest list
and store it at /etc/rescached/hosts.d/<name>, where name is the domain
name of source.

8 files changed, 395 insertions, 23 deletions

diff --git a/_www/src/App.svelte b/_www/src/App.svelte
index 3cde46c..833b470 100644
--- a/_www/src/App.svelte
+++ b/_www/src/App.svelte
@@ -1,16 +1,38 @@
 <script>
+	import { onMount } from 'svelte';
+
+	import { apiEnvironment, environment, nanoSeconds } from './environment.js';
 	import Environment from './Environment.svelte';
+	import HostsBlock from './HostsBlock.svelte';
+
+	const stateEnvironment = "environment"
+	const stateHostsBlock = "hosts_block"
 
 	export let name;
 	let state;
+	let env = {};
+
+	onMount(async () => {
+		const res = await fetch(apiEnvironment);
+		let got = await res.json();
+		got.PruneDelay = got.PruneDelay / nanoSeconds;
+		got.PruneThreshold = got.PruneThreshold / nanoSeconds;
+		env = Object.assign(env, got)
+		environment.set(env)
+		console.log("Environment:", environment)
+	});
 
 	function showEnvironment() {
-		if (state === 'environment') {
+		if (state === stateEnvironment) {
 			state = '';
 		} else {
-			state = 'environment';
+			state = stateEnvironment;
 		}
 	}
+
+	function showHostsBlock() {
+		state = stateHostsBlock
+	}
 </script>
 
 <style>
@@ -37,14 +59,19 @@
 <div class="main">
 	<h1> {name} </h1>
 	<nav class="menu">
-	<a href="#" on:click={showEnvironment}>
+	<a href="#environment" on:click={showEnvironment}>
 		Environment
 	</a>
+	/
+	<a href="#hostsblock" on:click={showHostsBlock}>
+		HostsBlock
+	</a>
 	</nav>
 
-	{#if state === 'environment'}
+	{#if state === stateEnvironment}
 		<Environment/>
-	{:else if state === 'hosts'}
+	{:else if state === stateHostsBlock}
+		<HostsBlock/>
 	{:else}
 		<p>
 			Welcome to rescached!
diff --git a/_www/src/Environment.svelte b/_www/src/Environment.svelte
index cab0c5f..ff9086c 100644
--- a/_www/src/Environment.svelte
+++ b/_www/src/Environment.svelte
@@ -1,24 +1,21 @@
 <script>
-	import { onMount } from 'svelte';
+	import { onDestroy } from 'svelte';
 
+	import { apiEnvironment, environment, nanoSeconds } from './environment.js';
 	import LabelHint from "./LabelHint.svelte";
 	import InputNumber from "./InputNumber.svelte";
 	import InputAddress from "./InputAddress.svelte";
 
-	const nanoSeconds = 1000000000;
-	let apiEnvironment = "http://127.0.0.1:5380/api/environment"
 	let env = {
 		NameServers: [],
 	};
 
-	onMount(async () => {
-		const res = await fetch(apiEnvironment);
-		let got = await res.json();
-		got.PruneDelay = got.PruneDelay / nanoSeconds;
-		got.PruneThreshold = got.PruneThreshold / nanoSeconds;
-		env = Object.assign(env, got)
+	const envUnsubscribe = environment.subscribe(value => {
+		env = value;
 	});
 
+	onDestroy(envUnsubscribe);
+
 	function addNameServer() {
 		env.NameServers = [...env.NameServers, '']
 	}
@@ -37,6 +34,8 @@
 		let got = {};
 
 		Object.assign(got, env)
+		environment.set(env)
+
 		got.PruneDelay = got.PruneDelay * nanoSeconds;
 		got.PruneThreshold = got.PruneThreshold * nanoSeconds;
 
diff --git a/_www/src/HostsBlock.svelte b/_www/src/HostsBlock.svelte
new file mode 100644
index 0000000..0b62544
--- /dev/null
+++ b/_www/src/HostsBlock.svelte
@@ -0,0 +1,102 @@
+<script>
+	import { onDestroy } from 'svelte';
+
+	import { apiEnvironment, environment, nanoSeconds } from './environment.js';
+
+	const apiHostsBlock = "/api/hosts_block"
+	let env = {};
+
+	const envUnsubscribe = environment.subscribe(value => {
+		env = value;
+	});
+	onDestroy(envUnsubscribe);
+
+	async function updateHostsBlocks() {
+		const res = await fetch(apiHostsBlock, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify(env.HostsBlocks),
+		});
+
+		const resJSON = await res.json()
+
+		console.log(resJSON);
+	}
+</script>
+
+<style>
+	.block_source.header {
+		font-weight: 600;
+	}
+	.block_source span {
+		font-size: 14px;
+		display: inline-block;
+		margin-right: 10px;
+		vertical-align: middle;
+	}
+	.block_source > span:nth-child(1) {
+		width: 60px;
+	}
+	.block_source > span:nth-child(2) {
+		width: 200px;
+	}
+	.block_source > span:nth-child(3) {
+		width: 300px;
+	}
+	.block_source > span:nth-child(3) input {
+		width: 300px;
+	}
+	.block_source > span:nth-child(4) {
+	}
+	.block_source input:disabled {
+		color: black;
+	}
+</style>
+
+<div class="hosts-block">
+	<h2>
+	/ Hosts block
+	</h2>
+
+	<p>
+	Configure the source of blocked hosts file.
+	</p>
+
+	<div class="block_source header">
+		<span> Enabled </span>
+		<span> Name </span>
+		<span> URL </span>
+		<span> Last updated </span>
+	</div>
+	<br/>
+	{#each env.HostsBlocks as hostsBlock}
+	<div class="block_source">
+		<span>
+			<input
+				type=checkbox
+				bind:checked={hostsBlock.IsEnabled}
+			>
+		</span>
+		<span>
+			{hostsBlock.Name}
+		</span>
+		<span>
+			<input
+				bind:value={hostsBlock.URL}
+				disabled
+			>
+		</span>
+		<span>
+			{hostsBlock.LastUpdated}
+		</span>
+	</div>
+	{/each}
+
+	<div>
+		<button on:click={updateHostsBlocks}>
+			Save
+		</button>
+	</div>
+</div>
diff --git a/_www/src/environment.js b/_www/src/environment.js
new file mode 100644
index 0000000..66bd74c
--- /dev/null
+++ b/_www/src/environment.js
@@ -0,0 +1,5 @@
+import { writable } from "svelte/store"
+
+export const apiEnvironment = "/api/environment"
+export const environment = writable({})
+export const nanoSeconds = 1000000000
diff --git a/cmd/rescached/rescached.cfg b/cmd/rescached/rescached.cfg
index 0df343b..93ffb22 100644
--- a/cmd/rescached/rescached.cfg
+++ b/cmd/rescached/rescached.cfg
@@ -5,10 +5,13 @@
 ##
 
 [rescached]
-dir.hosts=
-dir.master=
 file.resolvconf=
-debug=0
+debug=3
+
+hosts_block = http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&startdate[day]=&startdate[month]=&startdate[year]=&mimetype=plaintext
+hosts_block = http://www.malwaredomainlist.com/hostslist/hosts.txt
+hosts_block = http://winhelp2002.mvps.org/hosts.txt
+hosts_block = http://someonewhocares.org/hosts/hosts
 
 [dns "server"]
 #parent=udp://18.136.35.199
diff --git a/environment.go b/environment.go
index 1dfe834..ac72dec 100644
--- a/environment.go
+++ b/environment.go
@@ -6,9 +6,9 @@ package rescached
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"strconv"
+	"strings"
 
 	"github.com/shuLhan/share/lib/debug"
 	"github.com/shuLhan/share/lib/dns"
@@ -33,7 +33,11 @@ const (
 	keyCachePruneDelay     = "cache.prune_delay"
 	keyCachePruneThreshold = "cache.prune_threshold"
 	keyDohBehindProxy      = "doh.behind_proxy"
+	keyHostsBlock          = "hosts_block"
 	keyHTTPPort            = "http.port"
+	keyIsEnabled           = "is_enabled"
+	keyIsSystem            = "is_system"
+	keyLastUpdated         = "last_updated"
 	keyListen              = "listen"
 	keyParent              = "parent"
 	keyTLSAllowInsecure    = "tls.allow_insecure"
@@ -50,9 +54,11 @@ const (
 //
 type environment struct {
 	dns.ServerOptions
-	WuiListen      string `ini:"rescached::wui.listen"`
-	FileResolvConf string `ini:"rescached::file.resolvconf"`
-	Debug          int    `ini:"rescached::debug"`
+	WuiListen      string   `ini:"rescached::wui.listen"`
+	FileResolvConf string   `ini:"rescached::file.resolvconf"`
+	Debug          int      `ini:"rescached::debug"`
+	HostsBlocksRaw []string `ini:"rescached::hosts_block" json:"-"`
+	HostsBlocks    []*hostsBlock
 }
 
 func loadEnvironment(file string) (env *environment) {
@@ -63,19 +69,20 @@ func loadEnvironment(file string) (env *environment) {
 		return env
 	}
 
-	cfg, err := ioutil.ReadFile(file)
+	cfg, err := ini.Open(file)
 	if err != nil {
 		log.Printf("rescached: loadEnvironment %q: %s", file, err)
 		return env
 	}
 
-	err = ini.Unmarshal(cfg, env)
+	err = cfg.Unmarshal(env)
 	if err != nil {
 		log.Printf("rescached: loadEnvironment %q: %s", file, err)
 		return env
 	}
 
 	env.init()
+	env.initHostsBlock(cfg)
 	debug.Value = env.Debug
 
 	return env
@@ -107,6 +114,18 @@ func (env *environment) init() {
 	}
 }
 
+func (env *environment) initHostsBlock(cfg *ini.Ini) {
+	env.HostsBlocks = hostsBlockSources
+
+	for x, v := range env.HostsBlocksRaw {
+		env.HostsBlocksRaw[x] = strings.ToLower(v)
+	}
+
+	for _, hb := range env.HostsBlocks {
+		hb.init(env.HostsBlocksRaw)
+	}
+}
+
 func (env *environment) loadResolvConf() (ok bool, err error) {
 	rc, err := libnet.NewResolvConf(env.FileResolvConf)
 	if err != nil {
@@ -150,6 +169,13 @@ func (env *environment) write(file string) (err error) {
 	in.Set(sectionNameRescached, "", keyFileResolvConf, env.FileResolvConf)
 	in.Set(sectionNameRescached, "", keyDebug, strconv.Itoa(env.Debug))
 
+	in.UnsetAll(sectionNameRescached, "", keyHostsBlock)
+	for _, hb := range env.HostsBlocks {
+		if hb.IsEnabled {
+			in.Add(sectionNameRescached, "", keyHostsBlock, hb.URL)
+		}
+	}
+
 	in.UnsetAll(sectionNameDNS, subNameServer, keyParent)
 	for _, ns := range env.NameServers {
 		in.Add(sectionNameDNS, subNameServer, keyParent, ns)
diff --git a/hostsblock.go b/hostsblock.go
new file mode 100644
index 0000000..7a82c12
--- /dev/null
+++ b/hostsblock.go
@@ -0,0 +1,145 @@
+// Copyright 2020, Shulhan <ms@kilabit.info>. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package rescached
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+//
+// List of blocked hosts sources.
+//
+var hostsBlockSources = []*hostsBlock{{
+	Name: "pgl.yoyo.org",
+	URL:  `http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&startdate[day]=&startdate[month]=&startdate[year]=&mimetype=plaintext`,
+}, {
+	Name: "www.malwaredomainlist.com",
+	URL:  `http://www.malwaredomainlist.com/hostslist/hosts.txt`,
+}, {
+	Name: "winhelp2002.mvps.org",
+	URL:  `http://winhelp2002.mvps.org/hosts.txt`,
+}, {
+	Name: "someonewhocares.org",
+	URL:  `http://someonewhocares.org/hosts/hosts`,
+}}
+
+type hostsBlock struct {
+	Name        string // Derived from hostname in URL.
+	URL         string
+	LastUpdated time.Time
+	IsEnabled   bool
+	file        string
+}
+
+func (hb *hostsBlock) init(sources []string) {
+	for _, src := range sources {
+		if hb.URL == src {
+			hb.IsEnabled = true
+			break
+		}
+	}
+
+	// Set the LastUpdated from cache.
+	hb.file = filepath.Join(dirHosts, hb.Name)
+	fi, err := os.Stat(hb.file)
+	if err != nil {
+		return
+	}
+
+	hb.LastUpdated = fi.ModTime()
+}
+
+func (hb *hostsBlock) update(sources []*hostsBlock) bool {
+	for _, src := range sources {
+		if hb.Name == src.Name {
+			hb.IsEnabled = src.IsEnabled
+			break
+		}
+	}
+	if !hb.IsEnabled {
+		hb.hide()
+		return false
+	}
+
+	hb.unhide()
+
+	if !hb.isOld() {
+		return false
+	}
+
+	fmt.Printf("hostsBlock: updating %q\n", hb.Name)
+
+	res, err := http.Get(hb.URL)
+	if err != nil {
+		log.Printf("hostsBlock.update %q: %s", hb.Name, err)
+		return false
+	}
+	defer func() {
+		err := res.Body.Close()
+		if err != nil {
+			log.Printf("hostsBlock.update %q: %s", hb.Name, err)
+		}
+	}()
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		log.Printf("hostsBlock.update %q: %s", hb.Name, err)
+		return false
+	}
+
+	body = bytes.ReplaceAll(body, []byte("\r\n"), []byte("\n"))
+
+	err = ioutil.WriteFile(hb.file, body, 0644)
+	if err != nil {
+		log.Printf("hostsBlock.update %q: %s", hb.Name, err)
+		return false
+	}
+
+	return true
+}
+
+func (hb *hostsBlock) hide() {
+	if hb.LastUpdated.IsZero() {
+		return
+	}
+
+	newFileName := filepath.Join(dirHosts, "."+hb.Name)
+	err := os.Rename(hb.file, newFileName)
+	if err != nil {
+		log.Printf("hostsBlock.hide %q: %s", hb.file, err)
+		return
+	}
+
+	hb.file = newFileName
+}
+
+func (hb *hostsBlock) isOld() bool {
+	oneWeek := 7 * 24 * time.Hour
+	lastWeek := time.Now().Add(-1 * oneWeek)
+
+	return hb.LastUpdated.Before(lastWeek)
+}
+
+func (hb *hostsBlock) unhide() {
+	if hb.LastUpdated.IsZero() {
+		return
+	}
+
+	newFileName := filepath.Join(dirHosts, hb.Name)
+	err := os.Rename(hb.file, newFileName)
+	if err != nil {
+		log.Printf("hostsBlock.unhide %q: %s", hb.file, err)
+		return
+	}
+
+	hb.file = newFileName
+}
diff --git a/rescached_httpd.go b/rescached_httpd.go
index c875ff1..29ecc5c 100644
--- a/rescached_httpd.go
+++ b/rescached_httpd.go
@@ -26,6 +26,7 @@ func (srv *Server) httpdInit() (err error) {
 			`.*\.css`,
 			`.*\.html`,
 			`.*\.js`,
+			`.*\.png`,
 		},
 		CORSAllowOrigins: []string{
 			"http://127.0.0.1:5000",
@@ -33,6 +34,7 @@ func (srv *Server) httpdInit() (err error) {
 		CORSAllowHeaders: []string{
 			http.HeaderContentType,
 		},
+		Development: srv.env.Debug >= 3,
 	}
 
 	srv.httpd, err = http.NewServer(env)
@@ -75,6 +77,19 @@ func (srv *Server) httpdRegisterEndpoints() (err error) {
 		return err
 	}
 
+	epAPIPostHostsBlock := &http.Endpoint{
+		Method:       http.RequestMethodPost,
+		Path:         "/api/hosts_block",
+		RequestType:  http.RequestTypeJSON,
+		ResponseType: http.ResponseTypeJSON,
+		Call:         srv.apiPostHostsBlock,
+	}
+
+	err = srv.httpd.RegisterEndpoint(epAPIPostHostsBlock)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -142,3 +157,53 @@ func (srv *Server) httpdAPIPostEnvironment(
 
 	return json.Marshal(res)
 }
+
+func (srv *Server) apiPostHostsBlock(
+	httpRes stdhttp.ResponseWriter, req *stdhttp.Request, reqBody []byte,
+) (
+	resBody []byte, err error,
+) {
+	hostsBlocks := make([]*hostsBlock, 0)
+
+	err = json.Unmarshal(reqBody, &hostsBlocks)
+	if err != nil {
+		return nil, err
+	}
+
+	res := &liberrors.E{
+		Code:    stdhttp.StatusOK,
+		Message: "Restarting DNS server",
+	}
+
+	for x, hb := range hostsBlocks {
+		fmt.Printf("apiPostHostsBlock[%d]: %+v\n", x, hb)
+	}
+
+	var mustRestart bool
+	for _, hb := range srv.env.HostsBlocks {
+		isUpdated := hb.update(hostsBlocks)
+		if isUpdated {
+			mustRestart = true
+		}
+	}
+
+	err = srv.env.write(srv.fileConfig)
+	if err != nil {
+		log.Println("apiPostHostsBlock:", err.Error())
+		res.Code = stdhttp.StatusInternalServerError
+		res.Message = err.Error()
+		return json.Marshal(res)
+	}
+
+	if mustRestart {
+		srv.Stop()
+		err = srv.Start()
+		if err != nil {
+			log.Println("apiPostHostsBlock:", err.Error())
+			res.Code = stdhttp.StatusInternalServerError
+			res.Message = err.Error()
+		}
+	}
+
+	return json.Marshal(res)
+}