adding read me and adding error handling

analytics.go

@@ -0,0 +1,363 @@
+package analytics
+
+import (
+	"bytes"
+	"compress/zlib"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type Analyzer interface {
+	Dashboard(w http.ResponseWriter, r *http.Request)
+	InsertRequest(r *http.Request)
+}
+
+type AnalyticsConfiguration struct {
+	HashIPSecret         string
+	GroupByURLSegment    int
+	EntriesByURLSegment  int
+	WriteScheduleSeconds int
+	Name                 string
+	Password             string
+	Directory            string
+	UserAgentBlackList   []string
+}
+
+type analytics struct {
+	HashIPSecret         string
+	groupBy              int
+	entriesBy            int
+	WriteScheduleSeconds int
+	Password             string
+	Name                 string
+	Directory            string
+	Mux                  *sync.RWMutex
+	logger               func(...interface{}) (int, error)
+	UserAgentBlackList   []string
+	IPEntries            map[string]map[string][]action
+}
+
+func NewAnalytics(config AnalyticsConfiguration, logger func(...interface{}) (int, error)) Analyzer {
+	if logger == nil {
+		logger = fmt.Println
+	}
+	ana := &analytics{
+		Name:                 config.Name,
+		Password:             config.Password,
+		groupBy:              config.GroupByURLSegment,
+		entriesBy:            config.EntriesByURLSegment,
+		HashIPSecret:         config.HashIPSecret,
+		WriteScheduleSeconds: config.WriteScheduleSeconds,
+		Directory:            config.Directory,
+		UserAgentBlackList:   config.UserAgentBlackList,
+		Mux:                  &sync.RWMutex{},
+		logger:               logger,
+	}
+	ana.IPEntries = map[string]map[string][]action{}
+	ana.IPEntries[time.Now().Local().Format("2006-01-02")] = ana.readSavedData(time.Now().Local())
+	ana.scheduleWrite()
+	return ana
+}
+
+func (a analytics) scheduleWrite() {
+	ticker := time.NewTicker(time.Duration(a.WriteScheduleSeconds) * time.Second)
+	quit := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				err := a.writeFile()
+				if err != nil {
+					a.logger(err)
+				}
+			case <-quit:
+				ticker.Stop()
+				return
+			}
+		}
+	}()
+}
+
+var DefaultUserAgentBlacklist = []string{
+	"wget", "python", "perl", "msnbot", "netresearch", "bot",
+	"archive", "crawl", "googlebot", "msn", "archive", "php",
+	"panscient", "berry", "yandex", "bing", "fluffy",
+}
+
+func (a analytics) InsertRequest(r *http.Request) {
+	ua := strings.ToLower(r.UserAgent())
+	bots := a.UserAgentBlackList
+	for _, b := range bots {
+		if strings.Contains(strings.ToLower(ua), b) {
+			return
+		}
+	}
+	act := action{Page: r.URL.Path, Query: r.URL.RawQuery}
+	a.Mux.Lock()
+	defer a.Mux.Unlock()
+	a.insert(r.RemoteAddr, act)
+}
+
+func (a analytics) Dashboard(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	if len(a.Password) > 0 && (len(q["k"]) == 0 || len(q["k"][0]) == 0 || q["k"][0] != a.Password) {
+		a.logger(fmt.Errorf("Unauthorized"))
+		w.WriteHeader(http.StatusUnauthorized)
+		w.Write(nil)
+		return
+	}
+
+	date := time.Now()
+	var err error
+	if len(q["date"]) > 0 {
+		date, err = time.Parse("2006-01-02", q["date"][0])
+		if err != nil {
+			a.logger(err)
+			w.WriteHeader(http.StatusBadRequest)
+			w.Write(nil)
+			return
+		}
+	}
+	var data map[string][]action
+	if date.Format("2006-01-02") == time.Now().Format("2006-01-02") {
+		data = a.IPEntries[date.Format("2006-01-02")]
+	} else {
+		data = a.readSavedData(date)
+	}
+
+	entries := len(data)
+	urlHits := map[string]map[string]int{}
+	for _, actions := range data {
+		for _, act := range actions {
+			pParts := strings.Split(act.Page, "/")
+			groupBy := pParts[a.groupBy]
+			dataEntry := strings.Join(pParts[a.entriesBy:], "/")
+			_, ok := urlHits[groupBy]
+			if !ok {
+				urlHits[groupBy] = map[string]int{}
+			}
+
+			urlHits[groupBy][dataEntry] = urlHits[groupBy][dataEntry] + 1
+		}
+	}
+
+	dd := dashData{SessionCount: entries, URLHits: urlHits, Date: date.Format("2006-01-02")}
+	t, err := template.New("").Parse(HTML)
+	if err != nil {
+		a.logger(err)
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write(nil)
+		return
+	}
+	err = t.ExecuteTemplate(w, "layout", dd)
+	if err != nil {
+		a.logger(err)
+	}
+}
+
+type dashData struct {
+	Title        string
+	Content      string
+	CanonicalURL string
+	FootingQuote string
+	SessionCount int
+	Date         string
+	URLHits      map[string]map[string]int
+}
+
+type fileData struct {
+	Date    string
+	Entries map[string][]action
+}
+
+type action struct {
+	Page  string
+	Query string
+}
+
+func (a analytics) readSavedData(td time.Time) map[string][]action {
+	fileName := a.Directory + td.Format("/2006/01/02/") + a.Name + td.Format("2006-01-02")
+
+	entries := map[string][]action{}
+	if _, err := os.Stat(fileName); os.IsNotExist(err) {
+
+	} else {
+		bs, err := ioutil.ReadFile(fileName)
+		if err != nil {
+			a.logger(err)
+			return entries
+		}
+		r, err := zlib.NewReader(bytes.NewReader(bs))
+		if err != nil {
+			a.logger(err)
+			return entries
+		}
+		jsonBytes := bytes.NewBuffer([]byte{})
+		_, err = io.Copy(jsonBytes, r)
+		if err != nil {
+			a.logger(err)
+			return entries
+		}
+		r.Close()
+		err = json.Unmarshal(jsonBytes.Bytes(), &entries)
+		if err != nil {
+			a.logger(err)
+		}
+	}
+	return entries
+}
+
+func (a analytics) insert(ip string, act action) {
+	ts := time.Now().Format("2006-01-02")
+	stamps := a.IPEntries[ts]
+	if stamps == nil {
+		a.IPEntries[ts] = map[string][]action{}
+	}
+	if len(a.HashIPSecret) > 0 {
+		hash := sha256.New()
+		ip = ts + ip + a.HashIPSecret
+		inpIP := strings.NewReader(ip)
+		if _, err := io.Copy(hash, inpIP); err != nil {
+			a.logger(err)
+		}
+		sum := hash.Sum(nil)
+		ip = string(sum)
+	}
+	entries := stamps[ip]
+	if entries == nil {
+		entries = []action{}
+	}
+	entries = append(entries, act)
+
+	a.IPEntries[ts][ip] = entries
+}
+
+func (a analytics) writeFile() error {
+	ts := time.Now().Format("/2006/01/02")
+	err := os.MkdirAll(a.Directory+ts, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	a.Mux.Lock()
+	defer a.Mux.Unlock()
+	for k, e := range a.IPEntries {
+		data, err := json.Marshal(e)
+		if err != nil {
+			return err
+		}
+		f, err := os.Create(a.Directory + ts + "/" + a.Name + k)
+		if err != nil {
+			return err
+		}
+		var b bytes.Buffer
+		w := zlib.NewWriter(&b)
+		w.Write(data)
+		w.Close()
+		defer f.Close()
+		_, err = f.Write(b.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+const HTML = `
+{{ define "layout" }}
+<!DOCTYPE html>
+<html lang="en">
+    <head></head>
+    <body>
+        <style type="text/css">
+            .tg  {border-collapse:collapse;border-spacing:0;}
+            .tg td{border-color:black;border-style:solid;border-width:1px;font-family:Arial, sans-serif;font-size:14px; overflow:hidden;padding:10px 5px;word-break:normal;}
+            .tg th{border-color:black;border-style:solid;border-width:1px;font-family:Arial, sans-serif;font-size:14px; font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;}
+            .tg .tg-0lax{text-align:left;vertical-align:top}
+        </style>
+        <script>
+           function UpdateQueryString(key, value, url) {
+                if (!url) url = window.location.href;
+                var re = new RegExp("([?&])" + key + "=.*?(&|#|$)(.*)", "gi"),
+                    hash;
+
+                if (re.test(url)) {
+                    if (typeof value !== 'undefined' && value !== null) {
+                        return url.replace(re, '$1' + key + "=" + value + '$2$3');
+                    } 
+                    else {
+                        hash = url.split('#');
+                        url = hash[0].replace(re, '$1$3').replace(/(&|\?)$/, '');
+                        if (typeof hash[1] !== 'undefined' && hash[1] !== null) {
+                            url += '#' + hash[1];
+                        }
+                        return url;
+                    }
+                }
+                else {
+                    if (typeof value !== 'undefined' && value !== null) {
+                        var separator = url.indexOf('?') !== -1 ? '&' : '?';
+                        hash = url.split('#');
+                        url = hash[0] + separator + key + '=' + value;
+                        if (typeof hash[1] !== 'undefined' && hash[1] !== null) {
+                            url += '#' + hash[1];
+                        }
+                        return url;
+                    }
+                    else {
+                        return url;
+                    }
+                }
+            }
+
+            function chooseDate(object) {
+               window.location.href = UpdateQueryString("date", object.value, window.location.href)
+            }
+        </script>
+        <section id="about">
+            <div class="container-fluid align-self-center">
+                <div class="row d-flex justify-content-center">
+                    <div class="col-12 text-center align-self-center">
+                        <h1>{{.Date}}</h1>
+                        <input type="date" id="date" value="{{.Date}}" onchange="chooseDate(this)">
+                        <h2>Unique Sessions Today: {{.SessionCount}}</h2>
+                        <h3>Page Views</h3>
+                        {{range $Category, $URLS := .URLHits}}
+                            <h5> /{{$Category}}</h5>
+                            <table class="tg" style="undefined;table-layout: fixed; width: 320px">
+                                <colgroup>
+                                    <col style="width: 70px">
+                                    <col style="width: 250px">
+                                </colgroup>
+                                <thead>
+                                    <tr>
+                                        <th class="tg-0lax">Page Views</th>
+                                        <th class="tg-0lax">URL</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                {{range $URL, $count := $URLS}}
+                                    <tr>
+                                            <td class="tg-0lax">{{$count}} </td>
+                                            <td class="tg-0lax">{{$URL}}</td>
+                                    </tr>
+                                {{end}}
+                                </tbody>
+                            </table>
+                        {{ end }}
+                    </div>
+                </div>
+            </div>
+        </section>
+    </body>
+</html>
+{{ end }}
+`

analytics_mock.go

@@ -0,0 +1,47 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: analytics.go
+
+// Package mock_main is a generated GoMock package.
+package analytics
+
+import (
+	http "net/http"
+	reflect "reflect"
+
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockAnalyzer is a mock of Analyzer interface.
+type MockAnalyzer struct {
+	ctrl     *gomock.Controller
+	recorder *MockAnalyzerMockRecorder
+}
+
+// MockAnalyzerMockRecorder is the mock recorder for MockAnalyzer.
+type MockAnalyzerMockRecorder struct {
+	mock *MockAnalyzer
+}
+
+// NewMockAnalyzer creates a new mock instance.
+func NewMockAnalyzer(ctrl *gomock.Controller) *MockAnalyzer {
+	mock := &MockAnalyzer{ctrl: ctrl}
+	mock.recorder = &MockAnalyzerMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockAnalyzer) EXPECT() *MockAnalyzerMockRecorder {
+	return m.recorder
+}
+
+// InsertRequest mocks base method.
+func (m *MockAnalyzer) InsertRequest(r *http.Request) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "InsertRequest", r)
+}
+
+// InsertRequest indicates an expected call of InsertRequest.
+func (mr *MockAnalyzerMockRecorder) InsertRequest(r interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertRequest", reflect.TypeOf((*MockAnalyzer)(nil).InsertRequest), r)
+}

content/analytics.html

@@ -0,0 +1,88 @@
+{{ define "layout" }}
+<!DOCTYPE html>
+<html lang="en">
+    <head></head>
+    <body>
+        <style type="text/css">
+            .tg  {border-collapse:collapse;border-spacing:0;}
+            .tg td{border-color:black;border-style:solid;border-width:1px;font-family:Arial, sans-serif;font-size:14px; overflow:hidden;padding:10px 5px;word-break:normal;}
+            .tg th{border-color:black;border-style:solid;border-width:1px;font-family:Arial, sans-serif;font-size:14px; font-weight:normal;overflow:hidden;padding:10px 5px;word-break:normal;}
+            .tg .tg-0lax{text-align:left;vertical-align:top}
+        </style>
+        <script>
+           function UpdateQueryString(key, value, url) {
+                if (!url) url = window.location.href;
+                var re = new RegExp("([?&])" + key + "=.*?(&|#|$)(.*)", "gi"),
+                    hash;
+
+                if (re.test(url)) {
+                    if (typeof value !== 'undefined' && value !== null) {
+                        return url.replace(re, '$1' + key + "=" + value + '$2$3');
+                    } 
+                    else {
+                        hash = url.split('#');
+                        url = hash[0].replace(re, '$1$3').replace(/(&|\?)$/, '');
+                        if (typeof hash[1] !== 'undefined' && hash[1] !== null) {
+                            url += '#' + hash[1];
+                        }
+                        return url;
+                    }
+                }
+                else {
+                    if (typeof value !== 'undefined' && value !== null) {
+                        var separator = url.indexOf('?') !== -1 ? '&' : '?';
+                        hash = url.split('#');
+                        url = hash[0] + separator + key + '=' + value;
+                        if (typeof hash[1] !== 'undefined' && hash[1] !== null) {
+                            url += '#' + hash[1];
+                        }
+                        return url;
+                    }
+                    else {
+                        return url;
+                    }
+                }
+            }
+
+            function chooseDate(object) {
+               window.location.href = UpdateQueryString("date", object.value, window.location.href)
+            }
+        </script>
+        <section id="about">
+            <div class="container-fluid align-self-center">
+                <div class="row d-flex justify-content-center">
+                    <div class="col-12 text-center align-self-center">
+                        <h1>{{.Date}}</h1>
+                        <input type="date" id="date" value="{{.Date}}" onchange="chooseDate(this)">
+                        <h2>Unique Sessions Today: {{.SessionCount}}</h2>
+                        <h3>Page Views</h3>
+                        {{range $Category, $URLS := .URLHits}}
+                            <h5> /{{$Category}}</h5>
+                            <table class="tg" style="undefined;table-layout: fixed; width: 320px">
+                                <colgroup>
+                                    <col style="width: 70px">
+                                    <col style="width: 250px">
+                                </colgroup>
+                                <thead>
+                                    <tr>
+                                        <th class="tg-0lax">Page Views</th>
+                                        <th class="tg-0lax">URL</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                {{range $URL, $count := $URLS}}
+                                    <tr>
+                                            <td class="tg-0lax">{{$count}} </td>
+                                            <td class="tg-0lax">{{$URL}}</td>
+                                    </tr>
+                                {{end}}
+                                </tbody>
+                            </table>
+                        {{ end }}
+                    </div>
+                </div>
+            </div>
+        </section>
+    </body>
+</html>
+{{ end }}

go.mod

@@ -0,0 +1,5 @@
+module github.com/JakeKalstad/go-web-analytics
+
+go 1.17
+
+require github.com/golang/mock v1.6.0

go.sum

@@ -0,0 +1,25 @@
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

readme.md

@@ -0,0 +1,57 @@
+# go-web-analytics
+
+A minimal analytics package to start collecting traffic data without client dependencies.
+
+# Logging incoming requests
+
+    analytics := NewAnalytics(AnalyticsConfiguration{
+    			Name:                 "sanjuanpuertorico",
+    			Password:             os.Getenv("DASHBOARD_KEY"),
+    			GroupByURLSegment:    1,
+    			EntriesByURLSegment:  2,
+    			WriteScheduleSeconds: 30,
+    			Directory:            "logs",
+    			HashIPSecret:         os.Getenv("HASH_IP_KEY"),
+    			UserAgentBlackList:   DefaultUserAgentBlacklist,
+    		}, fmt.Println)
+
+
+    router.Use(func(next http.Handler) http.Handler {
+    	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+    		analytics.InsertRequest(r)
+    		next.ServeHTTP(w, r)
+    	})
+    })
+
+# Dashboard
+
+    router.HandleFunc("/analytics", analytics.Dashboard).Methods("GET")
+
+# Configuration
+
+    type AnalyticsConfiguration struct {
+        HashIPSecret         string
+        GroupByURLSegment    int
+        EntriesByURLSegment  int
+        WriteScheduleSeconds int
+        Name                 string
+        Password             string
+        Directory            string
+        UserAgentBlackList   []string
+    }
+
+> `HashIPSecret` is a seed that if provided will be used to hash 
+> the IP so you don't have plaintext user IPs stored
+
+> `GroupByURLSegment` index in the URL split by `/` to group the results
+
+> `EntriesByURLSegment` index in the URL split by `/` to count as results
+
+> `WriteScheduleSeconds` how often we write to the file
+> Name of file 
+
+> `Directory` parent directory for the log files
+
+> `Password` for a dashboard if it's used /analytics?k=mypassword
+
+> `UserAgentBlacklist` entries to check if the user agent contains in order to avoid things like bots or automated tests