flounder

A simple gemini site builder
Log | Files | Refs | README | LICENSE

commit 3628ff50dc1ea6fc7d858b2e20b69a811613ff26
parent 08060d4cf56be09b945bce5847b41db3d668b683
Author: alex wennerberg <awennerb@twitch.tv>
Date:   Mon,  8 Nov 2021 01:02:43 -0500

Remove log dumper

sqlite was a stupid way to do this. was fine for when there was less
traffic on this

Diffstat:
Mlog.go | 114-------------------------------------------------------------------------------
Mmain.go | 8+-------
2 files changed, 1 insertion(+), 121 deletions(-)

diff --git a/log.go b/log.go @@ -1,18 +1,11 @@ package main import ( - "bufio" - "database/sql" - "fmt" "io" - "log" "net" "net/http" "net/url" - "os" - "regexp" "strconv" - "strings" "time" "unicode/utf8" @@ -153,110 +146,3 @@ func appendQuoted(buf []byte, s string) []byte { } return buf } - -// notall fields set for both protocols -type LogLine struct { - Timestamp time.Time - Protocol string // gemini or http - ReqIP string // maybe rename here - ReqUser string - Status int - DestHost string - Method string - Referer string - Path string -} - -func (ll *LogLine) insertInto(db *sql.DB) { - _, err := db.Exec(`insert into log (timestamp, protocol, request_ip, request_user, status, destination_host, path, method, referer) -values (?, ?, ?, ?, ?, ?, ?, ?, ?)`, ll.Timestamp.Format(time.RFC3339), ll.Protocol, ll.ReqIP, ll.ReqUser, ll.Status, ll.DestHost, ll.Path, ll.Method, ll.Referer) - if err != nil { - fmt.Println(err) - } -} - -// TODO remove this junk, just use prometheus -const httpLogRegex = `^(.*?) - (.*?) \[(.*?)\] (.*?) \"(.*) (.*) .*\" - (.*) - (\d*)` - -var rxHttp *regexp.Regexp = regexp.MustCompile(httpLogRegex) - -func lineToLogLine(line string) (*LogLine, error) { - result := LogLine{} - var ts string - if strings.HasPrefix(line, "gemini") { - // nul - } else { - matches := rxHttp.FindStringSubmatch(line) - if len(matches) < 8 { - return nil, nil - } else { - result.ReqIP = matches[1] - result.ReqUser = matches[2] - ts = matches[3] - result.Timestamp, _ = time.Parse(apacheTS, ts) - result.DestHost = matches[4] - result.Method = matches[5] - result.Path = matches[6] - result.Referer = matches[7] - result.Status, _ = strconv.Atoi(matches[8]) - result.Protocol = "http" - } - } - return &result, nil -} - -func dumpLogsWorker() { - log.Println("Starting analytics log writing worker") - for { - dumpLogs() - time.Sleep(time.Minute * 5) - } -} -func dumpLogs() { - log.Println("Writing missing logs to database") - db, err := getAnalyticsDB() - if err != nil { - // not perfect -- squashes errors - return - } - var maxTime string - row := db.QueryRow(`SELECT timestamp from log order by timestamp desc limit 1`) - err = row.Scan(&maxTime) - if err != nil { - // not perfect -- squashes errors - return - } - - file, err := os.Open(c.LogFile) - if err != nil { - // not perfect -- squashes errors - return - } - defer file.Close() - - scanner := bufio.NewScanner(file) - counter := 0 - for scanner.Scan() { - text := scanner.Text() - ll, _ := lineToLogLine(text) - if ll == nil { - continue - } - if maxTime != "" { - max, err := time.Parse(time.RFC3339, maxTime) // ineff - if !ll.Timestamp.After(max) || err != nil { - // NOTE -- possible bug if two requests in the same second while we are reading -- skips 1 log - continue - } - } - ll.insertInto(db) - counter += 1 - } - log.Printf("Wrote %d logs\n", counter) -} - -func rotateLogs() { - // TODO write - // move log to log.1 - // delete log.1 -} diff --git a/main.go b/main.go @@ -44,11 +44,6 @@ func main() { // load domains in memory refreshDomainMap() - // Background workers - if c.AnalyticsDBFile != "" { - go dumpLogsWorker() - } - switch args[0] { case "serve": // s1.StartAsync() @@ -73,7 +68,6 @@ func main() { wg.Wait() case "admin": runAdminCommand() - case "dumplogs": - dumpLogs() + } }