add csv parsing; separate browser functions

Author: Ted Pearson

go.mod

@@ -3,12 +3,13 @@ module electric-usage-downloader
 go 1.19
 
 require (
+	github.com/chromedp/cdproto v0.0.0-20220924210414-0e3390be1777
 	github.com/chromedp/chromedp v0.8.6
+	github.com/shopspring/decimal v1.3.1
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/chromedp/cdproto v0.0.0-20220924210414-0e3390be1777 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect

go.sum

@@ -16,6 +16,8 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kUL
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
 golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/app/browser.go

@@ -0,0 +1,65 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/chromedp/cdproto/browser"
+	"github.com/chromedp/chromedp"
+)
+
+type Config struct {
+	Username    string
+	Password    string
+	LoginUrl    string
+	DownloadDir string
+}
+
+func DownloadCsv(config *Config, startDate string, endDate string) (string, error) {
+	allocatorFlags := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.Flag("headless", false))
+	ctx, cancel := chromedp.NewExecAllocator(context.Background(), allocatorFlags...)
+	defer cancel()
+	ctx, cancel = chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf))
+	defer cancel()
+	ctx, cancel = context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+
+	done := make(chan string, 1)
+	chromedp.ListenTarget(ctx, func(ev interface{}) {
+		if ev, ok := ev.(*browser.EventDownloadProgress); ok {
+			if ev.TotalBytes != 0 {
+				//fmt.Printf("State: %s, completed: %.2f, total: %.2f\n", ev.State.String(), ev.ReceivedBytes, ev.TotalBytes)
+				if ev.State == browser.DownloadProgressStateCompleted {
+					done <- ev.GUID
+					close(done)
+				}
+			}
+		}
+	})
+
+	err := chromedp.Run(ctx,
+		chromedp.Navigate(config.LoginUrl),
+		chromedp.SetValue("#LoginUsernameTextBox", config.Username, chromedp.NodeVisible),
+		chromedp.SetValue("#LoginPasswordTextBox", config.Password),
+		chromedp.Click("#LoginSubmitButton"),
+		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).
+			WithDownloadPath(config.DownloadDir).
+			WithEventsEnabled(true),
+		chromedp.Click("#MyUsageDropDown > a", chromedp.NodeVisible),
+		chromedp.Click(`//div[.="Usage Explorer"]`, chromedp.NodeVisible),
+		chromedp.Click(`//img[@alt='Usage Management']`, chromedp.NodeVisible),
+		chromedp.Sleep(time.Second),
+		chromedp.Click(`(//input[@name="timeFrameRadio"])[3]`, chromedp.NodeVisible),
+		chromedp.SetValue(`(//input[contains(@class, "form-control-readonly")])[1]`, startDate),
+		chromedp.SetValue(`(//input[contains(@class, "form-control-readonly")])[2]`, endDate),
+		chromedp.Click(`(//input[@name="fileFormatRadio"])[2]`),
+		chromedp.Click(`//button[.="Download Usage Data"]`),
+	)
+	if err != nil {
+		return "", err
+	}
+	guid := <-done
+	return fmt.Sprintf("%s/%s", config.DownloadDir, guid), nil
+}

internal/app/parser.go

@@ -0,0 +1,74 @@
+package app
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/shopspring/decimal"
+)
+
+type ElectricUsage struct {
+	StartTime   time.Time
+	EndTime     time.Time
+	WattHours   int64
+	CostInCents int64
+}
+
+type ElectricRecords struct {
+	ElectricUsage []*ElectricUsage
+}
+
+func ParseCsv(file string) (*ElectricRecords, error) {
+	dateRegex, err := regexp.Compile(`(\d{4}-\d\d-\d\d \d\d:\d\d) to (\d{4}-\d\d-\d\d \d\d:\d\d)`)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// drop all lines until it starts with dddd-dd-dd
+	data, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := data.Close(); err != nil {
+			panic(err)
+		}
+	}()
+	scanner := bufio.NewScanner(data)
+
+	records := make([]*ElectricUsage, 0, 100)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := dateRegex.FindStringSubmatch(line)
+		if matches != nil {
+			cols := strings.Split(line, ",")
+			usage := parseRecord(cols, matches[1:3])
+			records = append(records, usage)
+		}
+	}
+	return &ElectricRecords{records}, nil
+}
+
+func parseRecord(row []string, dates []string) *ElectricUsage {
+	if len(row) < 3 {
+		log.Printf("Bad record: %s\n", row)
+		return nil
+	}
+	startTime, err1 := time.ParseInLocation("2006-01-02 15:04", dates[0], time.Local)
+	endTime, err2 := time.ParseInLocation("2006-01-02 15:04", dates[1], time.Local)
+	kilowattHours, err3 := decimal.NewFromString(row[1])
+	cents, err4 := decimal.NewFromString(row[2])
+	if err1 != nil || err2 != nil || err3 != nil || err4 != nil {
+		log.Printf("Bad record: %s\n", row)
+		return nil
+	}
+	return &ElectricUsage{
+		StartTime:   startTime,
+		EndTime:     endTime,
+		WattHours:   kilowattHours.Mul(decimal.NewFromInt(1000)).IntPart(),
+		CostInCents: cents.Mul(decimal.NewFromInt(100)).IntPart(),
+	}
+}

main.go

@@ -1,81 +1,41 @@
 package main
 
 import (
-	"context"
 	"fmt"
-	"github.com/chromedp/cdproto/browser"
-	"github.com/chromedp/chromedp"
-	"gopkg.in/yaml.v3"
 	"log"
 	"os"
-)
 
-const loginUrl string = "https://novec.smarthub.coop/Login.html"
+	"gopkg.in/yaml.v3"
+
+	"electric-usage-downloader/internal/app"
+)
 
 var (
 	startDate = "09/29/2022"
 	endDate   = "09/30/2022"
 )
 
-type Config struct {
-	Username string
-	Password string
-}
-
 func main() {
-
 	// read config
 	file, err := os.ReadFile("config.yaml")
 	if err != nil {
 		log.Fatal(err)
 	}
-	config := &Config{}
+	config := &app.Config{}
 	err = yaml.Unmarshal(file, config)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	ctx, cancel := chromedp.NewExecAllocator(context.Background(), append(chromedp.DefaultExecAllocatorOptions[:], chromedp.Flag("headless", false))...)
-	defer cancel()
-
-	ctx, cancel = chromedp.NewContext(ctx, chromedp.WithLogf(log.Printf))
-	defer cancel()
-
-	//ctx, cancel = context.WithTimeout(ctx, 15*time.Second)
-	//defer cancel()
-
-	done := make(chan string, 1)
-	chromedp.ListenTarget(ctx, func(ev interface{}) {
-		if ev, ok := ev.(*browser.EventDownloadProgress); ok {
-			if ev.TotalBytes != 0 {
-				fmt.Printf("State: %s, completed: %.2f, total: %.2f\n", ev.State.String(), ev.ReceivedBytes, ev.TotalBytes)
-				if ev.State == browser.DownloadProgressStateCompleted {
-					done <- ev.GUID
-					close(done)
-				}
-			}
-		}
-	})
-
-	chromedp.Run(ctx,
-		chromedp.Navigate(loginUrl),
-		//chromedp.WaitVisible("#LoginUsernameTextBox"),
-		chromedp.SetValue("#LoginUsernameTextBox", config.Username, chromedp.NodeVisible),
-		chromedp.SetValue("#LoginPasswordTextBox", config.Password),
-		chromedp.Click("#LoginSubmitButton"),
-		//chromedp.WaitVisible(`//a[. = "Log Out"]`),
-		browser.SetDownloadBehavior(browser.SetDownloadBehaviorBehaviorAllowAndName).
-			WithDownloadPath("/tmp").
-			WithEventsEnabled(true),
-		chromedp.Click("#MyUsageDropDown > a", chromedp.NodeVisible),
-		chromedp.Click(`//div[.="Usage Explorer"]`, chromedp.NodeVisible),
-		chromedp.Click(`//img[@alt='Usage Management']`, chromedp.NodeVisible),
-		chromedp.Click(`(//input[@name="timeFrameRadio"])[3]`, chromedp.NodeVisible),
-		chromedp.SetValue(`(//input[contains(@class, "form-control-readonly")])[1]`, startDate),
-		chromedp.SetValue(`(//input[contains(@class, "form-control-readonly")])[2]`, endDate),
-		chromedp.Click(`(//input[@name="fileFormatRadio"])[2]`),
-		chromedp.Click(`//button[.="Download Usage Data"]`),
-	)
-	guid := <-done
-	fmt.Printf("hello %s", guid)
+	path, err := app.DownloadCsv(config, startDate, endDate)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("file downloaded: %s", path)
+	// parse csv!
+	records, err := app.ParseCsv(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%+v", records)
 }