diff options
| -rw-r--r-- | haminer.go | 9 | ||||
| -rw-r--r-- | http_log.go | 306 | ||||
| -rw-r--r-- | http_log_test.go | 45 | ||||
| -rw-r--r-- | testdata/ParseUDPPacket_test.txt | 37 |
4 files changed, 227 insertions, 170 deletions
@@ -118,9 +118,6 @@ func (h *Haminer) Start() (err error) { // filter will return true if log is accepted; otherwise it will return false. func (h *Haminer) filter(halog *HTTPLog) bool { - if halog == nil { - return false - } if halog.BackendName == `-` { return false } @@ -153,10 +150,8 @@ func (h *Haminer) consume() { continue } - halog = &HTTPLog{} - - ok = halog.ParseUDPPacket(packet[:n], h.cfg.RequestHeaders) - if !ok { + halog = ParseUDPPacket(packet[:n], h.cfg.RequestHeaders) + if halog == nil { continue } diff --git a/http_log.go b/http_log.go index bce442f..b90abb5 100644 --- a/http_log.go +++ b/http_log.go @@ -50,7 +50,8 @@ const ( type HTTPLog struct { Timestamp time.Time - RequestHeaders map[string]string + RequestHeaders map[string]string + ResponseHeaders map[string]string ClientIP string @@ -58,18 +59,19 @@ type HTTPLog struct { BackendName string ServerName string - CookieReq string - CookieRsp string - TermState string - HTTPMethod string HTTPURL string HTTPQuery string HTTPProto string tagHTTPURL string + CookieReq string + CookieRsp string + TermState string + BytesRead int64 + HTTPStatus int32 ClientPort int32 TimeReq int32 @@ -86,28 +88,152 @@ type HTTPLog struct { QueueServer int32 QueueBackend int32 +} - HTTPStatus int32 +// ParseUDPPacket convert UDP packet (in bytes) to instance of HTTPLog. +// +// It will return nil if UDP packet is nil, have zero length, or cannot be +// parsed (rejected). +func ParseUDPPacket(packet []byte, reqHeaders []string) (httpLog *HTTPLog) { + if len(packet) == 0 { + return nil + } + + if packet[0] == '<' { + var endIdx = bytes.IndexByte(packet, '>') + if endIdx < 0 { + return nil + } + packet = packet[endIdx+1:] + } + + return Parse(packet, reqHeaders) +} + +// Parse single line of HAProxy log format into HTTPLog. +// +// nolint: gocyclo +func Parse(in []byte, reqHeaders []string) (httpLog *HTTPLog) { + in = cleanPrefix(in) + if in == nil { + return nil + } + + var ok bool + + httpLog = &HTTPLog{} + + httpLog.ClientIP, ok = parseToString(in, ':') + if !ok { + return nil + } + + httpLog.ClientPort, ok = parseToInt32(in, ' ') + if !ok { + return nil + } + + // parse timestamp, remove '[' and parse until ']' + in = in[1:] + ts, ok := parseToString(in, ']') + if !ok { + return nil + } + + var err error + + httpLog.Timestamp, err = time.Parse(`2/Jan/2006:15:04:05.000`, ts) + if err != nil { + return nil + } + + in = in[1:] + httpLog.FrontendName, ok = parseToString(in, ' ') + if !ok { + return nil + } + + httpLog.BackendName, ok = parseToString(in, '/') + if !ok { + return nil + } + + httpLog.ServerName, ok = parseToString(in, ' ') + if !ok { + return nil + } + + ok = httpLog.parseConnectionTimes(in) + if !ok { + return nil + } + + httpLog.HTTPStatus, ok = parseToInt32(in, ' ') + if !ok { + return nil + } + + httpLog.BytesRead, ok = parseToInt64(in, ' ') + if !ok { + return nil + } + + httpLog.CookieReq, ok = parseToString(in, ' ') + if !ok { + return nil + } + httpLog.CookieRsp, ok = parseToString(in, ' ') + if !ok { + return nil + } + + httpLog.TermState, ok = parseToString(in, ' ') + if !ok { + return nil + } + + ok = httpLog.parseConns(in) + if !ok { + return nil + } + + ok = httpLog.parseQueue(in) + if !ok { + return nil + } + + if len(reqHeaders) > 0 { + ok = httpLog.parseRequestHeaders(in, reqHeaders) + if !ok { + return nil + } + } + + in = in[1:] + ok = httpLog.parseHTTP(in) + if !ok { + return nil + } + + return httpLog } -// cleanPrefix will remove `<date-time> <process-name>[pid]: ` prefix (which -// come from systemd/rsyslog) in input. -func cleanPrefix(in []byte) bool { - start := bytes.IndexByte(in, '[') +// cleanPrefix will remove `<date-time> <process-name>[pid]: ` prefix which +// come from systemd/rsyslog in input. +func cleanPrefix(in []byte) []byte { + var start = bytes.IndexByte(in, '[') if start < 0 { - return false + return nil } - end := bytes.IndexByte(in[start:], ']') + var end = bytes.IndexByte(in[start:], ']') if end < 0 { - return false + return nil } end = start + end + 3 - copy(in[0:], in[end:]) - - return true + return in[end:] } func parseToString(in []byte, sep byte) (string, bool) { @@ -165,7 +291,7 @@ func parseToInt64(in []byte, sep byte) (int64, bool) { return v, true } -func (httpLog *HTTPLog) parseTimes(in []byte) (ok bool) { +func (httpLog *HTTPLog) parseConnectionTimes(in []byte) (ok bool) { httpLog.TimeReq, ok = parseToInt32(in, '/') if !ok { return @@ -287,152 +413,6 @@ func (httpLog *HTTPLog) parseHTTP(in []byte) (ok bool) { return ok } -// Parse will parse one line of HAProxy log format into HTTPLog. -// -// nolint: gocyclo -func (httpLog *HTTPLog) Parse(in []byte, reqHeaders []string) (ok bool) { - var err error - - // Remove prefix from systemd/rsyslog - ok = cleanPrefix(in) - if !ok { - return - } - - // parse client IP - httpLog.ClientIP, ok = parseToString(in, ':') - if !ok { - return - } - - // parse client port - httpLog.ClientPort, ok = parseToInt32(in, ' ') - if !ok { - return - } - - // parse timestamp, remove '[' and parse until ']' - in = in[1:] - ts, ok := parseToString(in, ']') - if !ok { - return - } - - httpLog.Timestamp, err = time.Parse(`2/Jan/2006:15:04:05.000`, ts) - if err != nil { - return false - } - - // parse frontend name - in = in[1:] - httpLog.FrontendName, ok = parseToString(in, ' ') - if !ok { - return - } - - // parse backend name - httpLog.BackendName, ok = parseToString(in, '/') - if !ok { - return - } - - // parse server name - httpLog.ServerName, ok = parseToString(in, ' ') - if !ok { - return - } - - // parse times - ok = httpLog.parseTimes(in) - if !ok { - return - } - - // parse HTTP status code - httpLog.HTTPStatus, ok = parseToInt32(in, ' ') - if !ok { - return - } - - // parse bytes read - httpLog.BytesRead, ok = parseToInt64(in, ' ') - if !ok { - return - } - - // parse request cookie - httpLog.CookieReq, ok = parseToString(in, ' ') - if !ok { - return - } - - // parse response cookie - httpLog.CookieRsp, ok = parseToString(in, ' ') - if !ok { - return - } - - // parse termination state - httpLog.TermState, ok = parseToString(in, ' ') - if !ok { - return - } - - // parse number of connections - ok = httpLog.parseConns(in) - if !ok { - return - } - - // parse number of queue state - ok = httpLog.parseQueue(in) - if !ok { - return - } - - if len(reqHeaders) > 0 { - ok = httpLog.parseRequestHeaders(in, reqHeaders) - if !ok { - return - } - } - - // parse HTTP - in = in[1:] - ok = httpLog.parseHTTP(in) - - return ok -} - -// ParseUDPPacket will convert UDP packet (in bytes) to instance of -// HTTPLog. -// -// It will return nil and false if UDP packet is nil, have zero length, or -// cannot be parsed (rejected). -func (httpLog *HTTPLog) ParseUDPPacket(packet []byte, reqHeaders []string) bool { - if len(packet) == 0 { - return false - } - - var ( - endIdx int - in []byte - ) - - if packet[0] == '<' { - endIdx = bytes.IndexByte(packet, '>') - if endIdx < 0 { - return false - } - - in = packet[endIdx+1:] - } else { - in = packet - } - - return httpLog.Parse(in, reqHeaders) -} - // writeIlp write the HTTP log as Influxdb Line Protocol. func (httpLog *HTTPLog) writeIlp(out io.Writer) (err error) { var ( diff --git a/http_log_test.go b/http_log_test.go new file mode 100644 index 0000000..157ff71 --- /dev/null +++ b/http_log_test.go @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2018 M. Shulhan <ms@kilabit.info> +// SPDX-License-Identifier: GPL-3.0-or-later + +package haminer + +import ( + "encoding/json" + "testing" + + "git.sr.ht/~shulhan/pakakeh.go/lib/test" +) + +func TestParseUDPPacket(t *testing.T) { + var ( + logp = `TestParseUDPPacket` + tdata *test.Data + err error + ) + tdata, err = test.LoadData(`testdata/ParseUDPPacket_test.txt`) + if err != nil { + t.Fatal(logp, err) + } + + var listCase = []string{ + `http_log_0000`, + } + + var ( + httpLog *HTTPLog + tag string + exp string + got []byte + ) + for _, tag = range listCase { + httpLog = ParseUDPPacket(tdata.Input[tag], nil) + + got, err = json.MarshalIndent(httpLog, ``, ` `) + if err != nil { + t.Fatal(logp, err) + } + + exp = string(tdata.Output[tag]) + test.Assert(t, tag, exp, string(got)) + } +} diff --git a/testdata/ParseUDPPacket_test.txt b/testdata/ParseUDPPacket_test.txt new file mode 100644 index 0000000..9b93015 --- /dev/null +++ b/testdata/ParseUDPPacket_test.txt @@ -0,0 +1,37 @@ +Test data for ParseUDPPacket. + +>>> http_log_0000 +<134>Mar 17 05:08:28 haproxy[371]: 169.254.63.64:52722 [17/Mar/2024:05:08:28.886] fe-http be-http/be-http2 10/20/30/40/50 200 149 - - ---- 1/1/2/3/4 5/6 "GET / HTTP/1.1" + +<<< http_log_0000 +{ + "Timestamp": "2024-03-17T05:08:28.886Z", + "RequestHeaders": null, + "ResponseHeaders": null, + "ClientIP": "169.254.63.64", + "FrontendName": "fe-http", + "BackendName": "be-http", + "ServerName": "be-http2", + "HTTPMethod": "GET", + "HTTPURL": "/", + "HTTPQuery": "", + "HTTPProto": "HTTP/1.1", + "CookieReq": "-", + "CookieRsp": "-", + "TermState": "----", + "BytesRead": 149, + "HTTPStatus": 200, + "ClientPort": 52722, + "TimeReq": 10, + "TimeWait": 20, + "TimeConnect": 30, + "TimeRsp": 40, + "TimeAll": 50, + "ConnActive": 1, + "ConnFrontend": 1, + "ConnBackend": 2, + "ConnServer": 3, + "ConnRetries": 4, + "QueueServer": 5, + "QueueBackend": 6 +} |