all: implement UI to create, update, delete hosts file in hosts.d

The UI will update (insert or remove) the records on cache on the fly.

11 files changed, 588 insertions, 14 deletions

diff --git a/_www/src/App.svelte b/_www/src/App.svelte
index f86de2a..36c047c 100644
--- a/_www/src/App.svelte
+++ b/_www/src/App.svelte
@@ -4,9 +4,11 @@
 	import { apiEnvironment, environment, nanoSeconds } from './environment.js';
 	import Environment from './Environment.svelte';
 	import HostsBlock from './HostsBlock.svelte';
+	import HostsDir from './HostsDir.svelte';
 
 	const stateEnvironment = "environment"
 	const stateHostsBlock = "hosts_block"
+	const stateHostsDir = "hosts_d"
 
 	export let name;
 	let state;
@@ -18,8 +20,10 @@
 		got.PruneDelay = got.PruneDelay / nanoSeconds;
 		got.PruneThreshold = got.PruneThreshold / nanoSeconds;
 		env = Object.assign(env, got)
+        for (let x = 0; x < env.HostsFiles.length; x++) {
+            env.HostsFiles[x].hosts = [];
+        }
 		environment.set(env)
-		console.log("Environment:", environment)
 	});
 </script>
 
@@ -49,12 +53,18 @@
 	<a href="#{stateHostsBlock}" on:click={()=>state=stateHostsBlock}>
 		HostsBlock
 	</a>
+	/
+	<a href="#{stateHostsDir}" on:click={()=>state=stateHostsDir}>
+		hosts.d
+	</a>
 	</nav>
 
 	{#if state === stateEnvironment}
 		<Environment/>
 	{:else if state === stateHostsBlock}
 		<HostsBlock/>
+	{:else if state === stateHostsDir}
+		<HostsDir/>
 	{:else}
 		<p>
 			Welcome to rescached!
diff --git a/_www/src/HostsDir.svelte b/_www/src/HostsDir.svelte
new file mode 100644
index 0000000..b56f602
--- /dev/null
+++ b/_www/src/HostsDir.svelte
@@ -0,0 +1,189 @@
+<script>
+	import { onDestroy } from 'svelte';
+	import { apiEnvironment, environment, nanoSeconds } from './environment.js';
+
+	const apiHostsDir = "/api/hosts.d"
+	let env = {};
+    let hostsFile = {
+        Name: "",
+        hosts: [],
+    };
+    let newHostsFile = "";
+
+	const envUnsubscribe = environment.subscribe(value => {
+		env = value;
+	});
+	onDestroy(envUnsubscribe);
+
+    async function getHostsFile(hf) {
+        if (hf.hosts.length > 0) {
+            hostsFile = hf;
+            return;
+        }
+        const res = await fetch(apiHostsDir +"/"+ hf.Name);
+        hf.hosts = await res.json();
+        hostsFile = hf;
+    }
+
+    async function createHostsFile() {
+        if (newHostsFile === "") {
+            return;
+        }
+
+        const res = await fetch(apiHostsDir+ "/"+ newHostsFile, {
+            method: "PUT",
+        })
+
+        if (res.status >= 400) {
+            console.log("createHostsFile: ", res.status, res.statusText);
+            return;
+        }
+
+        const hf = {
+            Name: newHostsFile,
+            Path: newHostsFile,
+            hosts: [],
+        }
+        env.HostsFiles.push(hf);
+        env.HostsFiles = env.HostsFiles;
+    }
+
+    async function updateHostsFile() {
+        const res = await fetch(apiHostsDir+"/"+ hostsFile.Name, {
+            method: "POST",
+            body: JSON.stringify(hostsFile.hosts),
+        })
+
+        if (res.status >= 400) {
+            console.log("updateHostsFile: ", res.status, res.statusText);
+            return;
+        }
+
+        hostsFile.hosts = await res.json()
+    }
+
+    function addHost() {
+        let newHost = {
+            Name: "",
+            Value: "",
+        }
+        hostsFile.hosts.unshift(newHost);
+        hostsFile.hosts = hostsFile.hosts;
+    }
+
+    function deleteHost(idx) {
+        console.log('deleteHost at ', idx);
+        hostsFile.hosts.splice(idx, 1);
+        hostsFile.hosts = hostsFile.hosts;
+    }
+
+    async function deleteHostsFile(hfile) {
+        const res = await fetch(apiHostsDir+"/"+hfile.Name, {
+            method: "DELETE",
+        });
+        if (res.status >= 400) {
+            console.log("deleteHostsFile: ", res.status, res.statusText);
+            return;
+        }
+        for (let x = 0; x < env.HostsFiles.length; x++) {
+            if (env.HostsFiles[x].Name == hfile.Name) {
+                hostsFile = {Name: "", Path:"", hosts: []};
+                env.HostsFiles.splice(x, 1);
+                env.HostsFiles = env.HostsFiles;
+                return
+            }
+        }
+    }
+</script>
+
+<style>
+    .nav-left {
+        padding: 0px;
+        width: 300px;
+        float: left;
+    }
+    .nav-left .item {
+        margin: 4px 0px;
+    }
+    .content {
+        float: left;
+    }
+    .host {
+        font-family: monospace;
+        width: 100%;
+    }
+    input.host_name {
+        min-width: 240px;
+        width: calc(100% - 180px);
+    }
+    input.host_value {
+        width: 140px;
+    }
+</style>
+
+<div class="hosts_d">
+    <h2>
+        / hosts.d
+    </h2>
+
+    <div class="nav-left">
+        {#each env.HostsFiles as hf}
+        <div class="item">
+            <a href="#" on:click={getHostsFile(hf)}>
+                {hf.Name}
+            </a>
+        </div>
+        {/each}
+        <br/>
+        <label>
+            <span>New hosts file:</span>
+            <br/>
+            <input bind:value={newHostsFile}>
+        </label>
+        <button on:click={createHostsFile}>
+            Create
+        </button>
+    </div>
+
+    <div class="content">
+        {#if hostsFile.Name === ""}
+        <p>
+            Select one of the hosts file to manage.
+        </p>
+        {:else}
+        <p>
+            {hostsFile.Name} ({hostsFile.hosts.length} records)
+            <button on:click={deleteHostsFile(hostsFile)}>
+                Delete
+            </button>
+        </p>
+        <div>
+            <button on:click={addHost}>
+                Add
+            </button>
+        </div>
+
+        {#each hostsFile.hosts as host, idx (idx)}
+        <div class="host">
+            <input
+                class="host_name"
+                placeholder="Domain name"
+                bind:value={host.Name}
+            >
+            <input
+                class="host_value"
+                placeholder="IP address"
+                bind:value={host.Value}
+            >
+            <button on:click={deleteHost(idx)}>
+                X
+            </button>
+        </div>
+        {/each}
+
+        <button on:click={updateHostsFile}>
+            Save
+        </button>
+        {/if}
+    </div>
+</div>
diff --git a/cmd/rescached/_public b/cmd/rescached/_public
new file mode 120000
index 0000000..d56ea50
--- /dev/null
+++ b/cmd/rescached/_public
@@ -0,0 +1 @@
+../../_www/public
\ No newline at end of file
diff --git a/cmd/rescached/main.go b/cmd/rescached/main.go
index 9bc0be4..2675b02 100644
--- a/cmd/rescached/main.go
+++ b/cmd/rescached/main.go
@@ -39,7 +39,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	if debug.Value >= 3 {
+	if debug.Value >= 4 {
 		go debugRuntime()
 	}
 
diff --git a/environment.go b/environment.go
index a998884..34cb665 100644
--- a/environment.go
+++ b/environment.go
@@ -59,6 +59,7 @@ type environment struct {
 	Debug          int      `ini:"rescached::debug"`
 	HostsBlocksRaw []string `ini:"rescached::hosts_block" json:"-"`
 	HostsBlocks    []*hostsBlock
+	HostsFiles     []*hostsFile
 }
 
 func loadEnvironment(file string) (env *environment) {
diff --git a/go.mod b/go.mod
index 9f52862..c7a64c2 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,6 @@ module github.com/shuLhan/rescached-go/v3
 
 go 1.13
 
-require github.com/shuLhan/share v0.16.0
+require github.com/shuLhan/share v0.17.1-0.20200725201814-b5ac956c00fd
 
 //replace github.com/shuLhan/share => ../share
diff --git a/go.sum b/go.sum
index 6170af8..62cc6f6 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/shuLhan/share v0.16.0 h1:IldAfUlqf+csek1zpHzW2FJFuYm0tLYv8RrSyIkt5As=
-github.com/shuLhan/share v0.16.0/go.mod h1:FqPloTQlDTAmMXxaWft/V5tPmxEHBJeyJMAzVm4/1og=
+github.com/shuLhan/share v0.17.1-0.20200725201814-b5ac956c00fd h1:vZ4tbQWkXSrJ6d/2WfTbtnTXFmge7s1375ul5X/E8YQ=
+github.com/shuLhan/share v0.17.1-0.20200725201814-b5ac956c00fd/go.mod h1:FqPloTQlDTAmMXxaWft/V5tPmxEHBJeyJMAzVm4/1og=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
diff --git a/host.go b/host.go
new file mode 100644
index 0000000..521153d
--- /dev/null
+++ b/host.go
@@ -0,0 +1,34 @@
+// Copyright 2020, Shulhan <ms@kilabit.info>. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package rescached
+
+import "github.com/shuLhan/share/lib/dns"
+
+// host contains simplified DNS record.
+type host struct {
+	Name  string
+	Type  int
+	Class int
+	Value string
+	TTL   int
+}
+
+func convertRRToHost(from *dns.ResourceRecord) (to *host) {
+	to = &host{
+		Name:  string(from.Name),
+		Type:  int(from.Type),
+		Class: int(from.Class),
+		TTL:   int(from.TTL),
+	}
+	switch from.Type {
+	case dns.QueryTypeA, dns.QueryTypeNS, dns.QueryTypeCNAME,
+		dns.QueryTypeMB, dns.QueryTypeMG, dns.QueryTypeMR,
+		dns.QueryTypeNULL, dns.QueryTypePTR, dns.QueryTypeTXT,
+		dns.QueryTypeAAAA:
+		to.Value = string(from.RData().([]byte))
+	}
+
+	return to
+}
diff --git a/hosts_file.go b/hosts_file.go
new file mode 100644
index 0000000..c88f55e
--- /dev/null
+++ b/hosts_file.go
@@ -0,0 +1,128 @@
+// Copyright 2020, Shulhan <ms@kilabit.info>. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package rescached
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/shuLhan/share/lib/dns"
+)
+
+type hostsFile struct {
+	Name  string
+	Path  string
+	hosts []*host
+	out   *os.File
+}
+
+//
+// convertHostsFile convert the dns HostsFile to our hostsFile for simple
+// fetch and update.
+//
+func convertHostsFile(from *dns.HostsFile) (to *hostsFile) {
+	to = &hostsFile{
+		Name:  from.Name,
+		Path:  from.Path,
+		hosts: make([]*host, 0, len(from.Messages)),
+	}
+
+	for _, msg := range from.Messages {
+		if len(msg.Answer) == 0 {
+			continue
+		}
+
+		host := convertRRToHost(&msg.Answer[0])
+		if host != nil {
+			to.hosts = append(to.hosts, host)
+		}
+	}
+
+	return to
+}
+
+func newHostsFile(name string, hosts []*host) (hfile *hostsFile, err error) {
+	if hosts == nil {
+		hosts = make([]*host, 0)
+	}
+
+	hfile = &hostsFile{
+		Name:  name,
+		Path:  filepath.Join(dirHosts, name),
+		hosts: hosts,
+	}
+
+	hfile.out, err = os.OpenFile(hfile.Path, os.O_CREATE|os.O_RDWR, 0600)
+	if err != nil {
+		return nil, err
+	}
+
+	return hfile, nil
+}
+
+func (hfile *hostsFile) close() (err error) {
+	err = hfile.out.Close()
+	hfile.out = nil
+	return err
+}
+
+//
+// names return all hosts domain names.
+//
+func (hfile *hostsFile) names() (names []string) {
+	names = make([]string, 0, len(hfile.hosts))
+	for _, host := range hfile.hosts {
+		names = append(names, host.Name)
+	}
+	return names
+}
+
+func (hfile *hostsFile) update(hosts []*host) (msgs []*dns.Message, err error) {
+	if hfile.out == nil {
+		hfile.out, err = os.OpenFile(hfile.Path,
+			os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		err = hfile.out.Truncate(0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	hfile.hosts = hfile.hosts[:0]
+
+	for _, host := range hosts {
+		if len(host.Name) == 0 || len(host.Value) == 0 {
+			continue
+		}
+		msg := dns.NewMessageAddress(
+			[]byte(host.Name),
+			[][]byte{[]byte(host.Value)},
+		)
+		if msg == nil {
+			continue
+		}
+		_, err = fmt.Fprintf(hfile.out, "%s %s\n", host.Value, host.Name)
+		if err != nil {
+			return nil, err
+		}
+
+		// Make sure the Name is in lowercases.
+		host.Name = string(msg.Question.Name)
+
+		msgs = append(msgs, msg)
+		hfile.hosts = append(hfile.hosts, host)
+	}
+
+	err = hfile.close()
+	if err != nil {
+		return nil, err
+	}
+
+	return msgs, nil
+}
diff --git a/httpd.go b/httpd.go
index 29ecc5c..0b051ef 100644
--- a/httpd.go
+++ b/httpd.go
@@ -9,13 +9,15 @@ import (
 	"fmt"
 	"log"
 	stdhttp "net/http"
+	"os"
 
 	liberrors "github.com/shuLhan/share/lib/errors"
 	"github.com/shuLhan/share/lib/http"
 )
 
 const (
-	defHTTPDRootDir = "_www/public/"
+	defHTTPDRootDir = "_public/"
+	paramNameName   = "name"
 )
 
 func (srv *Server) httpdInit() (err error) {
@@ -90,6 +92,49 @@ func (srv *Server) httpdRegisterEndpoints() (err error) {
 		return err
 	}
 
+	err = srv.httpd.RegisterEndpoint(&http.Endpoint{
+		Method:       http.RequestMethodPut,
+		Path:         "/api/hosts.d/:name",
+		RequestType:  http.RequestTypeNone,
+		ResponseType: http.ResponseTypeNone,
+		Call:         srv.apiHostsFileCreate,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = srv.httpd.RegisterEndpoint(&http.Endpoint{
+		Method:       http.RequestMethodGet,
+		Path:         "/api/hosts.d/:name",
+		RequestType:  http.RequestTypeNone,
+		ResponseType: http.ResponseTypeJSON,
+		Call:         srv.apiHostsFileGet,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = srv.httpd.RegisterEndpoint(&http.Endpoint{
+		Method:       http.RequestMethodPost,
+		Path:         "/api/hosts.d/:name",
+		RequestType:  http.RequestTypeJSON,
+		ResponseType: http.ResponseTypeJSON,
+		Call:         srv.apiHostsFileUpdate,
+	})
+	if err != nil {
+		return err
+	}
+	err = srv.httpd.RegisterEndpoint(&http.Endpoint{
+		Method:       http.RequestMethodDelete,
+		Path:         "/api/hosts.d/:name",
+		RequestType:  http.RequestTypeNone,
+		ResponseType: http.ResponseTypeJSON,
+		Call:         srv.apiHostsFileDelete,
+	})
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -207,3 +252,167 @@ func (srv *Server) apiPostHostsBlock(
 
 	return json.Marshal(res)
 }
+
+func (srv *Server) apiHostsFileCreate(
+	httpres stdhttp.ResponseWriter, httpreq *stdhttp.Request, _ []byte,
+) (
+	resbody []byte, err error,
+) {
+	name := httpreq.Form.Get(paramNameName)
+	if len(name) == 0 {
+		return nil, &liberrors.E{
+			Code:    stdhttp.StatusBadRequest,
+			Message: "hosts file name is invalid or empty",
+		}
+	}
+
+	for _, hf := range srv.env.HostsFiles {
+		if hf.Name == name {
+			return nil, nil
+		}
+	}
+
+	hfile, err := newHostsFile(name, nil)
+	if err != nil {
+		return nil, &liberrors.E{
+			Code:    stdhttp.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+
+	err = hfile.close()
+	if err != nil {
+		return nil, &liberrors.E{
+			Code:    stdhttp.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+
+	srv.env.HostsFiles = append(srv.env.HostsFiles, hfile)
+
+	httpres.WriteHeader(stdhttp.StatusCreated)
+
+	return nil, nil
+}
+
+func (srv *Server) apiHostsFileGet(
+	_ stdhttp.ResponseWriter, httpreq *stdhttp.Request, _ []byte,
+) (
+	resbody []byte, err error,
+) {
+	hosts := make([]*host, 0)
+	name := httpreq.Form.Get(paramNameName)
+
+	for _, hfile := range srv.env.HostsFiles {
+		if hfile.Name == name {
+			hosts = hfile.hosts
+			break
+		}
+	}
+
+	return json.Marshal(&hosts)
+}
+
+func (srv *Server) apiHostsFileUpdate(
+	_ stdhttp.ResponseWriter, httpreq *stdhttp.Request, reqbody []byte,
+) (
+	resbody []byte, err error,
+) {
+	var (
+		hosts = make([]*host, 0)
+		name  = httpreq.Form.Get(paramNameName)
+		found bool
+		hfile *hostsFile
+	)
+
+	err = json.Unmarshal(reqbody, &hosts)
+	if err != nil {
+		return nil, &liberrors.E{
+			Code:    stdhttp.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+
+	for _, hfile = range srv.env.HostsFiles {
+		if hfile.Name == name {
+			found = true
+			break
+		}
+	}
+	if !found {
+		hfile, err = newHostsFile(name, hosts)
+		if err != nil {
+			return nil, &liberrors.E{
+				Code:    stdhttp.StatusInternalServerError,
+				Message: err.Error(),
+			}
+		}
+		srv.env.HostsFiles = append(srv.env.HostsFiles, hfile)
+	}
+
+	oldHostnames := hfile.names()
+
+	msgs, err := hfile.update(hosts)
+	if err != nil {
+		return nil, &liberrors.E{
+			Code:    stdhttp.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+
+	// Remove the records associated with hosts file.
+	srv.dns.RemoveCachesByNames(oldHostnames)
+
+	// Populate new hosts to cache.
+	srv.dns.PopulateCaches(msgs)
+
+	resbody, err = json.Marshal(&hfile.hosts)
+	if err != nil {
+		return nil, &liberrors.E{
+			Code:    stdhttp.StatusInternalServerError,
+			Message: err.Error(),
+		}
+	}
+
+	return resbody, nil
+}
+
+func (srv *Server) apiHostsFileDelete(
+	_ stdhttp.ResponseWriter, httpreq *stdhttp.Request, reqbody []byte,
+) (
+	resbody []byte, err error,
+) {
+	res := &liberrors.E{
+		Code: stdhttp.StatusOK,
+	}
+
+	name := httpreq.Form.Get(paramNameName)
+	if len(name) == 0 {
+		res.Message = "empty or invalid host file name"
+		return nil, res
+	}
+
+	for x, hfile := range srv.env.HostsFiles {
+		if hfile.Name != name {
+			continue
+		}
+
+		// Remove the records associated with hosts file.
+		srv.dns.RemoveCachesByNames(hfile.names())
+
+		err = os.RemoveAll(hfile.Path)
+		if err != nil {
+			res.Message = err.Error()
+			return nil, res
+		}
+
+		copy(srv.env.HostsFiles[x:], srv.env.HostsFiles[x+1:])
+		srv.env.HostsFiles[len(srv.env.HostsFiles)-1] = nil
+		srv.env.HostsFiles = srv.env.HostsFiles[:len(srv.env.HostsFiles)-1]
+
+		res.Message = name + " has been deleted"
+		return json.Marshal(res)
+	}
+	res.Message = "apiDeleteHostsFile: " + name + " not found"
+	return nil, res
+}
diff --git a/rescached.go b/rescached.go
index 3aab17a..6a74946 100644
--- a/rescached.go
+++ b/rescached.go
@@ -26,7 +26,6 @@ type Server struct {
 	httpd       *http.Server
 	httpdRunner sync.Once
 
-	hostsFiles  map[string]*dns.HostsFile
 	masterFiles map[string]*dns.MasterFile
 }
 
@@ -41,8 +40,9 @@ func New(fileConfig string) (srv *Server, err error) {
 	}
 
 	srv = &Server{
-		fileConfig: fileConfig,
-		env:        env,
+		fileConfig:  fileConfig,
+		env:         env,
+		masterFiles: make(map[string]*dns.MasterFile),
 	}
 
 	err = srv.httpdInit()
@@ -63,19 +63,21 @@ func (srv *Server) Start() (err error) {
 		return err
 	}
 
-	hostsFile, err := dns.ParseHostsFile(dns.GetSystemHosts())
+	dnsHostsFile, err := dns.ParseHostsFile(dns.GetSystemHosts())
 	if err != nil {
 		return err
 	}
-	srv.dns.PopulateCaches(hostsFile.Messages)
+	srv.dns.PopulateCaches(dnsHostsFile.Messages)
 
-	srv.hostsFiles, err = dns.LoadHostsDir(dirHosts)
+	dnsHostsFiles, err := dns.LoadHostsDir(dirHosts)
 	if err != nil {
 		return err
 	}
 
-	for _, hostsFile := range srv.hostsFiles {
-		srv.dns.PopulateCaches(hostsFile.Messages)
+	for _, dnsHostsFile := range dnsHostsFiles {
+		srv.dns.PopulateCaches(dnsHostsFile.Messages)
+		srv.env.HostsFiles = append(srv.env.HostsFiles,
+			convertHostsFile(dnsHostsFile))
 	}
 
 	srv.masterFiles, err = dns.LoadMasterDir(dirMaster)