fix time issues, just about prod ready

- don't allow date configurations that won't work
- fix tz issues with times and dates
- use UnixMilli() to compare with existing timestamps
- parse tz correctly from victoria metrics export output

Author: tedpearson

internal/app/main.go

@@ -43,44 +43,57 @@ func Main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	if config.ExtractDays > 45 || config.ExtractDays < 2 {
+		log.Fatal("ExtractDays must be between 2 and 45 per smarthub")
+	}
 
 	// parse time flags
 	startFlag := flag.String("start", "", "Start date of period to extract from electric co.")
 	endFlag := flag.String("end", "", "End date of period to extract from electric co.")
 	flag.Parse()
 	var startDate, endDate time.Time
 	if *startFlag != "" {
-		startDate, err = time.Parse("2006-01-02", *startFlag)
+		startDate, err = time.ParseInLocation("2006-01-02", *startFlag, time.Local)
 		if err != nil {
 			log.Fatal(err)
 		}
 		if *endFlag == "" {
 			log.Fatal("start and end parameters must both be provided")
 		}
-		endDate, err = time.Parse("2006-01-02", *endFlag)
+		endDate, err = time.ParseInLocation("2006-01-02", *endFlag, time.Local)
 		if err != nil {
 			log.Fatal(err)
 		}
+		if endDate.Sub(startDate).Hours() > 24*45 {
+			log.Fatal("start and end parameters must define a period of no more than 45 days")
+		}
+		// endDate should be the last minute of the day for the VictoriaMetrics query.
+		endDate = endDate.Add((24 * time.Hour) - time.Minute)
 	} else {
-		endDate = time.Now().Truncate(24 * time.Hour)
-		startDate = endDate.Add(time.Duration(-config.ExtractDays) * 24 * time.Hour)
+		// yesterday
+		year, month, day := time.Now().Date()
+		// endDate should be the last minute of the day for the VictoriaMetrics query.
+		endDate = time.Date(year, month, day, 23, 59, 0, 0, time.Local)
+		// subtract N days and 1 minute to get the start date
+		startDate = endDate.Add(time.Duration(-config.ExtractDays) * 48 * time.Hour).Add(time.Minute)
 	}
 
-	path, err := DownloadCsv(config, startDate.Format("01/02/2006"), endDate.Format("01/02/2002"))
+	path, err := DownloadCsv(config, startDate.Format("01/02/2006"), endDate.Format("01/02/2006"))
 	if err != nil {
 		log.Fatal(err)
 	}
 	fmt.Printf("file downloaded: %s", path)
 
-	// parse csv
 	records, err := ParseCsv(path)
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Printf("%+v", records)
 	existingPoints, err := QueryPreviousMetrics(startDate, endDate, config.InfluxDB)
 	if err != nil {
 		log.Fatal(err)
 	}
-	WriteMetrics(records, config.InfluxDB, existingPoints)
+	err = WriteMetrics(records, config.InfluxDB, existingPoints)
+	if err != nil {
+		log.Fatal(err)
+	}
 }

internal/app/metrics.go

@@ -17,14 +17,14 @@ type MetricLine struct {
 	Timestamps []int64
 }
 
-func WriteMetrics(records []*ElectricUsage, config InfluxDB, existingPoints map[int64]struct{}) {
+func WriteMetrics(records []*ElectricUsage, config InfluxDB, existingPoints map[int64]struct{}) error {
 	client := influxdb2.NewClient(config.Host, config.User+":"+config.Password)
 	writeApi := client.WriteAPIBlocking("", config.Database)
 	points := make([]*write.Point, 0, 15*2*len(records))
 	for _, record := range records {
 		divisor := record.EndTime.Sub(record.StartTime).Minutes()
 		for t := record.StartTime; record.EndTime.After(t); t = t.Add(time.Minute) {
-			if _, ok := existingPoints[t.Unix()]; ok {
+			if _, ok := existingPoints[t.UnixMilli()]; ok {
 				continue
 			}
 			watts := influxdb2.NewPointWithMeasurement("electric").
@@ -37,18 +37,15 @@ func WriteMetrics(records []*ElectricUsage, config InfluxDB, existingPoints map[
 		}
 	}
 
-	err := writeApi.WritePoint(context.Background(), points...)
-	if err != nil {
-		log.Fatal(err)
-	}
+	return writeApi.WritePoint(context.Background(), points...)
 }
 
 func QueryPreviousMetrics(startTime time.Time, endTime time.Time, config InfluxDB) (map[int64]struct{}, error) {
 	client := &http.Client{}
 	v := url.Values{
-		"match[]": {"sensor_temperature"},
-		"start":   {startTime.Format("2006-01-02T15:04:05+07:00")},
-		"end":     {endTime.Format("2006-01-02T15:04:05+07:00")},
+		"match[]": {"electric_usage"},
+		"start":   {startTime.Format(`2006-01-02T15:04:05Z07:00`)},
+		"end":     {endTime.Format("2006-01-02T15:04:05Z07:00")},
 	}
 	req, err := http.NewRequest("POST", config.Host+"/api/v1/export", strings.NewReader(v.Encode()))
 	if err != nil {