diff options
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | _database/0001_http_log.sql | 68 | ||||
| -rw-r--r-- | cmd/haminer/main.go | 7 | ||||
| -rw-r--r-- | haminer.go | 32 | ||||
| -rw-r--r-- | internal/cmd/memfs/main.go | 41 | ||||
| -rw-r--r-- | memfs_database.go | 80 |
6 files changed, 226 insertions, 7 deletions
@@ -4,7 +4,10 @@ .PHONY: all build lint serve-doc all: build lint test -build: +embed: + go run ./internal/cmd/memfs + +build: embed go build -o ./_bin/ ./cmd/... ##---- Run all tests and generate coverage as HTML. diff --git a/_database/0001_http_log.sql b/_database/0001_http_log.sql new file mode 100644 index 0000000..47da300 --- /dev/null +++ b/_database/0001_http_log.sql @@ -0,0 +1,68 @@ +DROP TABLE IF EXISTS http_log CASCADE; + +CREATE TABLE http_log ( + request_date TIMESTAMP WITH TIME ZONE + +, client_ip VARCHAR + +, frontend_name VARCHAR +, backend_name VARCHAR +, server_name VARCHAR + +, http_proto VARCHAR +, http_method VARCHAR +, http_url VARCHAR +, http_query VARCHAR + +, header_request VARCHAR +, header_response VARCHAR + +, cookie_request VARCHAR +, cookie_response VARCHAR +, termination_state VARCHAR + +, bytes_read BIGINT + +, status_code INTEGER +, client_port INTEGER + +, time_request INTEGER +, time_wait INTEGER +, time_connect INTEGER +, time_response INTEGER +, time_all INTEGER + +, conn_active INTEGER +, conn_frontend INTEGER +, conn_backend INTEGER +, conn_server INTEGER +, retries INTEGER + +, server_queue INTEGER +, backend_queue INTEGER +); + +DROP INDEX IF EXISTS http_log_idx; + +CREATE INDEX IF NOT EXISTS http_log_idx ON http_log( + request_date +, client_ip +, frontend_name +, backend_name +, server_name +, http_proto +, http_method +, http_url +, termination_state +, status_code +); + +DROP INDEX IF EXISTS http_log_time_idx; + +CREATE INDEX IF NOT EXISTS http_log_time_idx ON http_log( + time_request +, time_wait +, time_connect +, time_response +, time_all +); diff --git a/cmd/haminer/main.go b/cmd/haminer/main.go index 0a2d49d..ec7cd8b 100644 --- a/cmd/haminer/main.go +++ b/cmd/haminer/main.go @@ -43,7 +43,12 @@ func main() { fmt.Printf("Starting Haminer with config: %+v\n", cfg) - h := haminer.NewHaminer(cfg) + var h *haminer.Haminer + + h, err = haminer.NewHaminer(cfg) + if err != nil { + log.Fatal(err) + } signal.Notify(chSignal, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) @@ -9,6 +9,8 @@ import ( "net" "os" "time" + + "git.sr.ht/~shulhan/pakakeh.go/lib/memfs" ) const ( @@ -20,6 +22,11 @@ var ( _hostname string ) +// memfsDatabase embed all ".sql" files in directory _database. +// It will be used to migrate the database when using postgresql as +// forwarder. +var memfsDatabase *memfs.MemFS + // Haminer define the log consumer and producer. type Haminer struct { cfg *Config @@ -43,7 +50,9 @@ func initHostname() { // NewHaminer create, initialize, and return new Haminer instance. If config // parameter is nil, it will use the default options. -func NewHaminer(cfg *Config) (h *Haminer) { +func NewHaminer(cfg *Config) (h *Haminer, err error) { + var logp = `NewHaminer` + if cfg == nil { cfg = NewConfig() } @@ -56,18 +65,20 @@ func NewHaminer(cfg *Config) (h *Haminer) { initHostname() - h.createForwarder() + err = h.createForwarder() + if err != nil { + return nil, fmt.Errorf(`%s: %w`, logp, err) + } - return + return h, nil } -func (h *Haminer) createForwarder() { +func (h *Haminer) createForwarder() (err error) { var ( logp = `createForwarder` fwCfg *ConfigForwarder fwName string - err error ) for fwName, fwCfg = range h.cfg.Forwarders { @@ -94,6 +105,10 @@ func (h *Haminer) createForwarder() { h.ff = append(h.ff, questc) case forwarderKindPostgresql: + if fwCfg.URL == `` { + continue + } + var pgc *forwarderPostgresql pgc, err = newForwarderPostgresql(*fwCfg) @@ -101,9 +116,16 @@ func (h *Haminer) createForwarder() { log.Printf(`%s: %s: %s`, logp, fwName, err) continue } + + err = pgc.conn.Migrate(``, memfsDatabase) + if err != nil { + return fmt.Errorf(`%s: %w`, logp, err) + } + h.ff = append(h.ff, pgc) } } + return nil } // Start will listen for UDP packet and start consuming log, parse, and diff --git a/internal/cmd/memfs/main.go b/internal/cmd/memfs/main.go new file mode 100644 index 0000000..59b8fdd --- /dev/null +++ b/internal/cmd/memfs/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "log" + "os" + + "git.sr.ht/~shulhan/pakakeh.go/lib/memfs" +) + +func main() { + embedDatabase() +} + +func embedDatabase() { + var memfsOpts = memfs.Options{ + Embed: memfs.EmbedOptions{ + PackageName: `haminer`, + VarName: `memfsDatabase`, + GoFileName: `memfs_database.go`, + }, + Root: `_database`, + Includes: []string{ + `.*\.sql$`, + }, + } + + var ( + mfs *memfs.MemFS + err error + ) + + mfs, err = memfs.New(&memfsOpts) + if err != nil { + log.Fatal(os.Args[0], err) + } + + err = mfs.GoEmbed() + if err != nil { + log.Fatal(os.Args[0], err) + } +} diff --git a/memfs_database.go b/memfs_database.go new file mode 100644 index 0000000..001c8ec --- /dev/null +++ b/memfs_database.go @@ -0,0 +1,80 @@ +// Code generated by git.sr.ht/~shulhan/pakakeh.go/lib/memfs DO NOT EDIT. + +package haminer + +import ( + "git.sr.ht/~shulhan/pakakeh.go/lib/memfs" +) + +func generate__database() *memfs.Node { + var node = &memfs.Node{ + SysPath: "_database", + Path: "/", + ContentType: "", + GenFuncName: "generate__database", + } + node.SetMode(2147484141) + node.SetModTimeUnix(1710702526, 180685009) + node.SetName("/") + node.SetSize(0) + node.AddChild(_memfsDatabase_getNode(memfsDatabase, "/0001_http_log.sql", generate__database_0001_http_log_sql)) + return node +} + +func generate__database_0001_http_log_sql() *memfs.Node { + var node = &memfs.Node{ + SysPath: "_database/0001_http_log.sql", + Path: "/0001_http_log.sql", + ContentType: "application/sql", + GenFuncName: "generate__database_0001_http_log_sql", + Content: []byte("\x44\x52\x4F\x50\x20\x54\x41\x42\x4C\x45\x20\x49\x46\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x20\x43\x41\x53\x43\x41\x44\x45\x3B\x0A\x0A\x43\x52\x45\x41\x54\x45\x20\x54\x41\x42\x4C\x45\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x20\x28\x0A\x20\x20\x72\x65\x71\x75\x65\x73\x74\x5F\x64\x61\x74\x65\x20\x20\x54\x49\x4D\x45\x53\x54\x41\x4D\x50\x20\x57\x49\x54\x48\x20\x54\x49\x4D\x45\x20\x5A\x4F\x4E\x45\x0A\x0A\x2C\x20\x63\x6C\x69\x65\x6E\x74\x5F\x69\x70\x20\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x66\x72\x6F\x6E\x74\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x62\x61\x63\x6B\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x73\x65\x72\x76\x65\x72\x5F\x6E\x61\x6D\x65\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x68\x74\x74\x70\x5F\x70\x72\x6F\x74\x6F\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x74\x74\x70\x5F\x6D\x65\x74\x68\x6F\x64\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x74\x74\x70\x5F\x75\x72\x6C\x20\x20\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x74\x74\x70\x5F\x71\x75\x65\x72\x79\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x68\x65\x61\x64\x65\x72\x5F\x72\x65\x71\x75\x65\x73\x74\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x68\x65\x61\x64\x65\x72\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x63\x6F\x6F\x6B\x69\x65\x5F\x72\x65\x71\x75\x65\x73\x74\x20\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x63\x6F\x6F\x6B\x69\x65\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x20\x20\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x2C\x20\x74\x65\x72\x6D\x69\x6E\x61\x74\x69\x6F\x6E\x5F\x73\x74\x61\x74\x65\x20\x56\x41\x52\x43\x48\x41\x52\x0A\x0A\x2C\x20\x62\x79\x74\x65\x73\x5F\x72\x65\x61\x64\x20\x20\x20\x20\x42\x49\x47\x49\x4E\x54\x0A\x0A\x2C\x20\x73\x74\x61\x74\x75\x73\x5F\x63\x6F\x64\x65\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6C\x69\x65\x6E\x74\x5F\x70\x6F\x72\x74\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x72\x65\x71\x75\x65\x73\x74\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x77\x61\x69\x74\x20\x20\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x63\x6F\x6E\x6E\x65\x63\x74\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x61\x6C\x6C\x20\x20\x20\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x61\x63\x74\x69\x76\x65\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x66\x72\x6F\x6E\x74\x65\x6E\x64\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x62\x61\x63\x6B\x65\x6E\x64\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x63\x6F\x6E\x6E\x5F\x73\x65\x72\x76\x65\x72\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x72\x65\x74\x72\x69\x65\x73\x20\x20\x20\x20\x20\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x0A\x2C\x20\x73\x65\x72\x76\x65\x72\x5F\x71\x75\x65\x75\x65\x20\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x2C\x20\x62\x61\x63\x6B\x65\x6E\x64\x5F\x71\x75\x65\x75\x65\x20\x49\x4E\x54\x45\x47\x45\x52\x0A\x29\x3B\x0A\x0A\x44\x52\x4F\x50\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x69\x64\x78\x3B\x0A\x0A\x43\x52\x45\x41\x54\x45\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x4E\x4F\x54\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x69\x64\x78\x20\x4F\x4E\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x28\x0A\x20\x20\x72\x65\x71\x75\x65\x73\x74\x5F\x64\x61\x74\x65\x0A\x2C\x20\x63\x6C\x69\x65\x6E\x74\x5F\x69\x70\x0A\x2C\x20\x66\x72\x6F\x6E\x74\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x0A\x2C\x20\x62\x61\x63\x6B\x65\x6E\x64\x5F\x6E\x61\x6D\x65\x0A\x2C\x20\x73\x65\x72\x76\x65\x72\x5F\x6E\x61\x6D\x65\x0A\x2C\x20\x68\x74\x74\x70\x5F\x70\x72\x6F\x74\x6F\x0A\x2C\x20\x68\x74\x74\x70\x5F\x6D\x65\x74\x68\x6F\x64\x0A\x2C\x20\x68\x74\x74\x70\x5F\x75\x72\x6C\x0A\x2C\x20\x74\x65\x72\x6D\x69\x6E\x61\x74\x69\x6F\x6E\x5F\x73\x74\x61\x74\x65\x0A\x2C\x20\x73\x74\x61\x74\x75\x73\x5F\x63\x6F\x64\x65\x0A\x29\x3B\x0A\x0A\x44\x52\x4F\x50\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x74\x69\x6D\x65\x5F\x69\x64\x78\x3B\x0A\x0A\x43\x52\x45\x41\x54\x45\x20\x49\x4E\x44\x45\x58\x20\x49\x46\x20\x4E\x4F\x54\x20\x45\x58\x49\x53\x54\x53\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x5F\x74\x69\x6D\x65\x5F\x69\x64\x78\x20\x4F\x4E\x20\x68\x74\x74\x70\x5F\x6C\x6F\x67\x28\x0A\x20\x20\x74\x69\x6D\x65\x5F\x72\x65\x71\x75\x65\x73\x74\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x77\x61\x69\x74\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x63\x6F\x6E\x6E\x65\x63\x74\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x72\x65\x73\x70\x6F\x6E\x73\x65\x0A\x2C\x20\x74\x69\x6D\x65\x5F\x61\x6C\x6C\x0A\x29\x3B\x0A"), + } + node.SetMode(420) + node.SetModTimeUnix(1710768370, 528442449) + node.SetName("0001_http_log.sql") + node.SetSize(1216) + return node +} + +// _memfsDatabase_getNode is internal function to minimize duplicate node +// created on Node.AddChild() and on generatedPathNode.Set(). +func _memfsDatabase_getNode(mfs *memfs.MemFS, path string, fn func() *memfs.Node) (node *memfs.Node) { + node = mfs.PathNodes.Get(path) + if node != nil { + return node + } + return fn() +} + +func init() { + memfsDatabase = &memfs.MemFS{ + PathNodes: memfs.NewPathNode(), + Opts: &memfs.Options{ + Root: "_database", + MaxFileSize: 5242880, + Includes: []string{ + `.*\.sql$`, + }, + Excludes: []string{ + }, + Embed: memfs.EmbedOptions{ + CommentHeader: ``, + PackageName: "haminer", + VarName: "memfsDatabase", + GoFileName: "memfs_database.go", + WithoutModTime: false, + }, + }, + } + memfsDatabase.PathNodes.Set("/", + _memfsDatabase_getNode(memfsDatabase, "/", generate__database)) + memfsDatabase.PathNodes.Set("/0001_http_log.sql", + _memfsDatabase_getNode(memfsDatabase, "/0001_http_log.sql", generate__database_0001_http_log_sql)) + + memfsDatabase.Root = memfsDatabase.PathNodes.Get("/") + + var err = memfsDatabase.Init() + if err != nil { + panic("memfsDatabase: " + err.Error()) + } +} |