aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile5
-rw-r--r--_database/0001_http_log.sql68
-rw-r--r--cmd/haminer/main.go7
-rw-r--r--haminer.go32
-rw-r--r--internal/cmd/memfs/main.go41
-rw-r--r--memfs_database.go80
6 files changed, 226 insertions, 7 deletions
diff --git a/Makefile b/Makefile
index f840717..b903567 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,10 @@
.PHONY: all build lint serve-doc
all: build lint test
-build:
+embed:
+ go run ./internal/cmd/memfs
+
+build: embed
go build -o ./_bin/ ./cmd/...
##---- Run all tests and generate coverage as HTML.
diff --git a/_database/0001_http_log.sql b/_database/0001_http_log.sql
new file mode 100644
index 0000000..47da300
--- /dev/null
+++ b/_database/0001_http_log.sql
@@ -0,0 +1,68 @@
+DROP TABLE IF EXISTS http_log CASCADE;
+
+CREATE TABLE http_log (
+ request_date TIMESTAMP WITH TIME ZONE
+
+, client_ip VARCHAR
+
+, frontend_name VARCHAR
+, backend_name VARCHAR
+, server_name VARCHAR
+
+, http_proto VARCHAR
+, http_method VARCHAR
+, http_url VARCHAR
+, http_query VARCHAR
+
+, header_request VARCHAR
+, header_response VARCHAR
+
+, cookie_request VARCHAR
+, cookie_response VARCHAR
+, termination_state VARCHAR
+
+, bytes_read BIGINT
+
+, status_code INTEGER
+, client_port INTEGER
+
+, time_request INTEGER
+, time_wait INTEGER
+, time_connect INTEGER
+, time_response INTEGER
+, time_all INTEGER
+
+, conn_active INTEGER
+, conn_frontend INTEGER
+, conn_backend INTEGER
+, conn_server INTEGER
+, retries INTEGER
+
+, server_queue INTEGER
+, backend_queue INTEGER
+);
+
+DROP INDEX IF EXISTS http_log_idx;
+
+CREATE INDEX IF NOT EXISTS http_log_idx ON http_log(
+ request_date
+, client_ip
+, frontend_name
+, backend_name
+, server_name
+, http_proto
+, http_method
+, http_url
+, termination_state
+, status_code
+);
+
+DROP INDEX IF EXISTS http_log_time_idx;
+
+CREATE INDEX IF NOT EXISTS http_log_time_idx ON http_log(
+ time_request
+, time_wait
+, time_connect
+, time_response
+, time_all
+);
diff --git a/cmd/haminer/main.go b/cmd/haminer/main.go
index 0a2d49d..ec7cd8b 100644
--- a/cmd/haminer/main.go
+++ b/cmd/haminer/main.go
@@ -43,7 +43,12 @@ func main() {
fmt.Printf("Starting Haminer with config: %+v\n", cfg)
- h := haminer.NewHaminer(cfg)
+ var h *haminer.Haminer
+
+ h, err = haminer.NewHaminer(cfg)
+ if err != nil {
+ log.Fatal(err)
+ }
signal.Notify(chSignal, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
diff --git a/haminer.go b/haminer.go
index 64f47a8..eeec52b 100644
--- a/haminer.go
+++ b/haminer.go
@@ -9,6 +9,8 @@ import (
"net"
"os"
"time"
+
+ "git.sr.ht/~shulhan/pakakeh.go/lib/memfs"
)
const (
@@ -20,6 +22,11 @@ var (
_hostname string
)
+// memfsDatabase embed all ".sql" files in directory _database.
+// It will be used to migrate the database when using postgresql as
+// forwarder.
+var memfsDatabase *memfs.MemFS
+
// Haminer define the log consumer and producer.
type Haminer struct {
cfg *Config
@@ -43,7 +50,9 @@ func initHostname() {
// NewHaminer create, initialize, and return new Haminer instance. If config
// parameter is nil, it will use the default options.
-func NewHaminer(cfg *Config) (h *Haminer) {
+func NewHaminer(cfg *Config) (h *Haminer, err error) {
+ var logp = `NewHaminer`
+
if cfg == nil {
cfg = NewConfig()
}
@@ -56,18 +65,20 @@ func NewHaminer(cfg *Config) (h *Haminer) {
initHostname()
- h.createForwarder()
+ err = h.createForwarder()
+ if err != nil {
+ return nil, fmt.Errorf(`%s: %w`, logp, err)
+ }
- return
+ return h, nil
}
-func (h *Haminer) createForwarder() {
+func (h *Haminer) createForwarder() (err error) {
var (
logp = `createForwarder`
fwCfg *ConfigForwarder
fwName string
- err error
)
for fwName, fwCfg = range h.cfg.Forwarders {
@@ -94,6 +105,10 @@ func (h *Haminer) createForwarder() {
h.ff = append(h.ff, questc)
case forwarderKindPostgresql:
+ if fwCfg.URL == `` {
+ continue
+ }
+
var pgc *forwarderPostgresql
pgc, err = newForwarderPostgresql(*fwCfg)
@@ -101,9 +116,16 @@ func (h *Haminer) createForwarder() {
log.Printf(`%s: %s: %s`, logp, fwName, err)
continue
}
+
+ err = pgc.conn.Migrate(``, memfsDatabase)
+ if err != nil {
+ return fmt.Errorf(`%s: %w`, logp, err)
+ }
+
h.ff = append(h.ff, pgc)
}
}
+ return nil
}
// Start will listen for UDP packet and start consuming log, parse, and
diff --git a/internal/cmd/memfs/main.go b/internal/cmd/memfs/main.go
new file mode 100644
index 0000000..59b8fdd
--- /dev/null
+++ b/internal/cmd/memfs/main.go
@@ -0,0 +1,41 @@
+package main
+
+import (
+ "log"
+ "os"
+
+ "git.sr.ht/~shulhan/pakakeh.go/lib/memfs"
+)
+
+func main() {
+ embedDatabase()
+}
+
+func embedDatabase() {
+ var memfsOpts = memfs.Options{
+ Embed: memfs.EmbedOptions{
+ PackageName: `haminer`,
+ VarName: `memfsDatabase`,
+ GoFileName: `memfs_database.go`,
+ },
+ Root: `_database`,
+ Includes: []string{
+ `.*\.sql$`,
+ },
+ }
+
+ var (
+ mfs *memfs.MemFS
+ err error
+ )
+
+ mfs, err = memfs.New(&memfsOpts)
+ if err != nil {
+ log.Fatal(os.Args[0], err)
+ }
+
+ err = mfs.GoEmbed()
+ if err != nil {
+ log.Fatal(os.Args[0], err)
+ }
+}
diff --git a/memfs_database.go b/memfs_database.go
new file mode 100644
index 0000000..001c8ec
--- /dev/null
+++ b/memfs_database.go
@@ -0,0 +1,80 @@
+// Code generated by git.sr.ht/~shulhan/pakakeh.go/lib/memfs DO NOT EDIT.
+
+package haminer
+
+import (
+ "git.sr.ht/~shulhan/pakakeh.go/lib/memfs"
+)
+
+func generate__database() *memfs.Node {
+ var node = &memfs.Node{
+ SysPath: "_database",
+ Path: "/",
+ ContentType: "",
+ GenFuncName: "generate__database",
+ }
+ node.SetMode(2147484141)
+ node.SetModTimeUnix(1710702526, 180685009)
+ node.SetName("/")
+ node.SetSize(0)
+ node.AddChild(_memfsDatabase_getNode(memfsDatabase, "/0001_http_log.sql", generate__database_0001_http_log_sql))
+ return node
+}
+
+func generate__database_0001_http_log_sql() *memfs.Node {
+ var node = &memfs.Node{
+ SysPath: "_database/0001_http_log.sql",
+ Path: "/0001_http_log.sql",
+ ContentType: "application/sql",
+ GenFuncName: "generate__database_0001_http_log_sql",
+ Content: []byte("\x44\x52\x4F\x50\x20\x54\x41\x42\x4C\x45\x20\x49\x46\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x20\x43\x41\x53\x43\x41\x44\x45\x3B\x0A\x0A\x43\x52\x45\x41\x54\x45\x20\x54\x41\x42\x4C\x45\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x20\x28\x0A\x20\x20\x72\x65\x71\x75\x65\x73\x74\x5F\x64\x61\x74\x65\x20\x20\x54\x49\x4D\x45\x53\x54\x41\x4D\x50\x20\x57\x49\x54\x48\x20\x54\x49\x4D\x45\x20\x5A\x4F\x4E\x45\x0A\x0A\x2C\x20\x63\x6C\x69\x65\x6E\x74\x5F\x69\x70\x20\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x66\x72\x6F\x6E\x74\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x62\x61\x63\x6B\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x73\x65\x72\x76\x65\x72\x5F\x6E\x61\x6D\x65\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x68\x74\x74\x70\x5F\x70\x72\x6F\x74\x6F\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x74\x74\x70\x5F\x6D\x65\x74\x68\x6F\x64\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x74\x74\x70\x5F\x75\x72\x6C\x20\x20\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x74\x74\x70\x5F\x71\x75\x65\x72\x79\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x68\x65\x61\x64\x65\x72\x5F\x72\x65\x71\x75\x65\x73\x74\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x65\x61\x64\x65\x72\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x63\x6F\x6F\x6B\x69\x65\x5F\x72\x65\x71\x75\x65\x73\x74\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x63\x6F\x6F\x6B\x69\x65\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x74\x65\x72\x6D\x69\x6E\x61\x74\x69\x6F\x6E\x5F\x73\x74\x61\x74\x65\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x62\x79\x74\x65\x73\x5F\x72\x65\x61\x64\x20\x20\x20\x20\x42\x49\x47\x49\x4E\x54\x0A\x0A\x2C\x20\x73\x74\x61\x74\x75\x73\x5F\x63\x6F\x64\x65\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6C\x69\x65\x6E\x74\x5F\x70\x6F\x72\x74\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x72\x65\x71\x75\x65\x73\x74\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x77\x61\x69\x74\x20\x20\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x63\x6F\x6E\x6E\x65\x63\x74\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x61\x6C\x6C\x20\x20\x20\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x61\x63\x74\x69\x76\x65\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x66\x72\x6F\x6E\x74\x65\x6E\x64\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x62\x61\x63\x6B\x65\x6E\x64\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x73\x65\x72\x76\x65\x72\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x72\x65\x74\x72\x69\x65\x73\x20\x20\x20\x20\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x0A\x2C\x20\x73\x65\x72\x76\x65\x72\x5F\x71\x75\x65\x75\x65\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x62\x61\x63\x6B\x65\x6E\x64\x5F\x71\x75\x65\x75\x65\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x29\x3B\x0A\x0A\x44\x52\x4F\x50\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x69\x64\x78\x3B\x0A\x0A\x43\x52\x45\x41\x54\x45\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x4E\x4F\x54\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x69\x64\x78\x20\x4F\x4E\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x28\x0A\x20\x20\x72\x65\x71\x75\x65\x73\x74\x5F\x64\x61\x74\x65\x0A\x2C\x20\x63\x6C\x69\x65\x6E\x74\x5F\x69\x70\x0A\x2C\x20\x66\x72\x6F\x6E\x74\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x0A\x2C\x20\x62\x61\x63\x6B\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x0A\x2C\x20\x73\x65\x72\x76\x65\x72\x5F\x6E\x61\x6D\x65\x0A\x2C\x20\x68\x74\x74\x70\x5F\x70\x72\x6F\x74\x6F\x0A\x2C\x20\x68\x74\x74\x70\x5F\x6D\x65\x74\x68\x6F\x64\x0A\x2C\x20\x68\x74\x74\x70\x5F\x75\x72\x6C\x0A\x2C\x20\x74\x65\x72\x6D\x69\x6E\x61\x74\x69\x6F\x6E\x5F\x73\x74\x61\x74\x65\x0A\x2C\x20\x73\x74\x61\x74\x75\x73\x5F\x63\x6F\x64\x65\x0A\x29\x3B\x0A\x0A\x44\x52\x4F\x50\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x74\x69\x6D\x65\x5F\x69\x64\x78\x3B\x0A\x0A\x43\x52\x45\x41\x54\x45\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x4E\x4F\x54\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x74\x69\x6D\x65\x5F\x69\x64\x78\x20\x4F\x4E\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x28\x0A\x20\x20\x74\x69\x6D\x65\x5F\x72\x65\x71\x75\x65\x73\x74\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x77\x61\x69\x74\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x63\x6F\x6E\x6E\x65\x63\x74\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x61\x6C\x6C\x0A\x29\x3B\x0A"),
+ }
+ node.SetMode(420)
+ node.SetModTimeUnix(1710768370, 528442449)
+ node.SetName("0001_http_log.sql")
+ node.SetSize(1216)
+ return node
+}
+
+// _memfsDatabase_getNode is internal function to minimize duplicate node
+// created on Node.AddChild() and on generatedPathNode.Set().
+func _memfsDatabase_getNode(mfs *memfs.MemFS, path string, fn func() *memfs.Node) (node *memfs.Node) {
+ node = mfs.PathNodes.Get(path)
+ if node != nil {
+ return node
+ }
+ return fn()
+}
+
+func init() {
+ memfsDatabase = &memfs.MemFS{
+ PathNodes: memfs.NewPathNode(),
+ Opts: &memfs.Options{
+ Root: "_database",
+ MaxFileSize: 5242880,
+ Includes: []string{
+ `.*\.sql$`,
+ },
+ Excludes: []string{
+ },
+ Embed: memfs.EmbedOptions{
+ CommentHeader: ``,
+ PackageName: "haminer",
+ VarName: "memfsDatabase",
+ GoFileName: "memfs_database.go",
+ WithoutModTime: false,
+ },
+ },
+ }
+ memfsDatabase.PathNodes.Set("/",
+ _memfsDatabase_getNode(memfsDatabase, "/", generate__database))
+ memfsDatabase.PathNodes.Set("/0001_http_log.sql",
+ _memfsDatabase_getNode(memfsDatabase, "/0001_http_log.sql", generate__database_0001_http_log_sql))
+
+ memfsDatabase.Root = memfsDatabase.PathNodes.Get("/")
+
+ var err = memfsDatabase.Init()
+ if err != nil {
+ panic("memfsDatabase: " + err.Error())
+ }
+}
generated by cgit v1.3 (git 2.53.0) at 2026-04-15 03:08:29 +0000