flounder

A simple gemini site builder
Log | Files | Refs | README | LICENSE

commit cfafb492208f46510499512e66c64fbfd060af94
parent 422f8d44a496cf7d701c731615c8c6c043f3a9da
Author: alex wennerberg <alex@alexwennerberg.com>
Date:   Sat,  2 Jan 2021 13:07:21 -0800

Add analytics db

Diffstat:
Mconfig.go | 1+
Mdb.go | 19+++++++++++++++++++
Mgo.mod | 1+
Mgo.sum | 2++
Mlog.go | 115+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
Mmain.go | 11+++++++++++
6 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/config.go b/config.go @@ -18,6 +18,7 @@ type Config struct { Debug bool SecretKey string DBFile string + AnalyticsDBFile string LogFile string GeminiCertStore string CookieStoreKey string diff --git a/db.go b/db.go @@ -24,6 +24,25 @@ func initializeDB() { createTablesIfDNE() } +func getAnalyticsDB() *sql.DB { + db, err := sql.Open("sqlite3", c.AnalyticsDBFile) + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS log ( + id INTEGER PRIMARY KEY NOT NULL, + timestamp TEXT NOT NULL, + protocol TEXT NOT NULL, + request_ip TEXT, + request_user TEXT, + status INTEGER, + destination_host TEXT, + path TEXT, + method TEXT +);`) + if err != nil { + log.Fatal(err) + } + return db +} + type File struct { // also folders Creator string Name string // includes folder diff --git a/go.mod b/go.mod @@ -6,6 +6,7 @@ require ( git.sr.ht/~adnano/go-gemini v0.1.8 github.com/BurntSushi/toml v0.3.1 github.com/emersion/go-webdav v0.3.0 + github.com/go-co-op/gocron v0.5.0 github.com/gorilla/feeds v1.1.1 github.com/gorilla/handlers v1.5.1 github.com/gorilla/sessions v1.2.1 diff --git a/go.sum b/go.sum @@ -27,6 +27,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-co-op/gocron v0.5.0 h1:QBxhIsODdq6u9+Cu3JehdhznE2IzuEokOddpLnxVrtg= +github.com/go-co-op/gocron v0.5.0/go.mod h1:6Btk4lVj3bnFAgbVfr76W8impTyhYrEi1pV5Pt4Tp/M= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= diff --git a/log.go b/log.go @@ -1,6 +1,8 @@ package main import ( + "bufio" + "database/sql" "fmt" gmi "git.sr.ht/~adnano/go-gemini" "github.com/gorilla/handlers" @@ -9,7 +11,10 @@ import ( "net" "net/http" "net/url" + "os" + "regexp" "strconv" + "strings" "time" "unicode/utf8" ) @@ -18,6 +23,8 @@ import ( const lowerhex = "0123456789abcdef" +const apacheTS = "02/Jan/2006:15:04:05 -0700" + func logFormatter(writer io.Writer, params handlers.LogFormatterParams) { buf := buildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) buf = append(buf, '\n') @@ -64,7 +71,7 @@ func buildCommonLogLine(req *http.Request, url url.URL, ts time.Time, status int buf = append(buf, " - "...) buf = append(buf, username...) buf = append(buf, " ["...) - buf = append(buf, ts.Format("02/Jan/2006:15:04:05 -0700")...) + buf = append(buf, ts.Format(apacheTS)...) buf = append(buf, `] `...) buf = append(buf, desthost...) buf = append(buf, ` "`...) @@ -155,9 +162,113 @@ func logGemini(r *gmi.Request) { host = ipAddr } line := fmt.Sprintf("gemini %s - [%s] %s %s\n", host, - time.Now().Format("02/Jan/2006:15:04:05 -0700"), + time.Now().Format(apacheTS), r.URL.Host, r.URL.Path) buf := []byte(line) log.Writer().Write(buf) } + +// notall fields set for both protocols +type LogLine struct { + Timestamp time.Time + Protocol string // gemini or http + ReqIP string // maybe rename here + ReqUser string + Status int + DestHost string + Method string + Path string +} + +func (ll *LogLine) insertInto(db *sql.DB) { + _, err := db.Exec(`insert into log (timestamp, protocol, request_ip, request_user, status, destination_host, path, method) +values (?, ?, ?, ?, ?, ?, ?, ?)`, ll.Timestamp.Format(time.RFC3339), ll.Protocol, ll.ReqIP, ll.ReqUser, ll.Status, ll.DestHost, ll.Path, ll.Method) + if err != nil { + fmt.Println(err) + } +} + +const httpLogRegex = `^(.*?) - (.*?) \[(.*?)\] (.*?) \"(.*) (.*) .*\" (\d*)` +const geminiLogRegex = `^gemini (.*?) - \[(.*?)\] (.*?) (.*)` + +var rxHttp *regexp.Regexp = regexp.MustCompile(httpLogRegex) +var rxGemini *regexp.Regexp = regexp.MustCompile(geminiLogRegex) + +func lineToLogLine(line string) (*LogLine, error) { + result := LogLine{} + var ts string + if strings.HasPrefix(line, "gemini") { + matches := rxGemini.FindStringSubmatch(line) + if len(matches) < 5 { + return nil, nil // TODO better error + } else { + result.ReqIP = matches[1] + ts = matches[2] + result.Timestamp, _ = time.Parse(apacheTS, ts) + result.DestHost = matches[3] + result.Path = matches[4] + result.Protocol = "gemini" + // etc + } + } else { + matches := rxHttp.FindStringSubmatch(line) + if len(matches) < 8 { + return nil, nil + } else { + result.ReqIP = matches[1] + result.ReqUser = matches[2] + ts = matches[3] + result.Timestamp, _ = time.Parse(apacheTS, ts) + result.DestHost = matches[4] + result.Method = matches[5] + result.Path = matches[6] + result.Status, _ = strconv.Atoi(matches[7]) + result.Protocol = "http" + } + } + return &result, nil +} + +func dumpLogs() { + fmt.Println("Writing missing logs to database") + db := getAnalyticsDB() + var maxTime string + row := db.QueryRow(`SELECT timestamp from log order by timestamp desc limit 1`) + err := row.Scan(&maxTime) + if err != nil { + // not perfect -- squashes errors + } + + file, err := os.Open(c.LogFile) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + counter := 0 + for scanner.Scan() { + text := scanner.Text() + ll, _ := lineToLogLine(text) + if ll == nil { + continue + } + if maxTime != "" { + max, err := time.Parse(time.RFC3339, maxTime) // ineff + if !ll.Timestamp.After(max) || err != nil { + // NOTE -- possible bug if two requests in the same second while we are reading -- skips 1 log + continue + } + } + ll.insertInto(db) + counter += 1 + } + fmt.Printf("Wrote %d logs\n", counter) +} + +func rotateLogs() { + // TODO write + // move log to log.1 + // delete log.1 +} diff --git a/main.go b/main.go @@ -3,11 +3,13 @@ package main import ( "flag" "fmt" + "github.com/go-co-op/gocron" "github.com/gorilla/sessions" "io" "log" "os" "sync" + "time" ) var c Config // global var to hold static configuration @@ -46,8 +48,15 @@ func main() { cookie := generateCookieKeyIfDNE() SessionStore = sessions.NewCookieStore(cookie) + // handle background tasks + s1 := gocron.NewScheduler(time.UTC) + if c.AnalyticsDBFile != "" { + s1.Every(1).Day().Do(dumpLogs) // TODO Dont do on start? + } + switch args[0] { case "serve": + s1.StartAsync() wg := new(sync.WaitGroup) wg.Add(2) go func() { @@ -61,5 +70,7 @@ func main() { wg.Wait() case "admin": runAdminCommand() + case "dumplogs": + dumpLogs() } }