Collaboration

.dockerignore

@@ -0,0 +1,3 @@
+*.csv
+*.csv2
+.git

.gitignore

@@ -0,0 +1 @@
+extras/model.go

Dockerfile

@@ -5,7 +5,7 @@ RUN apk update && apk add --no-cache git
 RUN apk --no-cache add ca-certificates
 
 WORKDIR /app
-COPY . /app/
+COPY *.go /app/
 
 # Fetch dependencies.
 RUN go get -d -v
@@ -23,6 +23,7 @@ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 #COPY --from=builder /app/files/ITEMS.txt.gz /app/files/ITEMS.txt.gz
 
 WORKDIR /app
+
 # Run the binary.
 
 ENV http_db_host "0.0.0.0:8000"

README.md

@@ -1,32 +1,78 @@
 # LambdaDB
 In memory database that uses filters to get the data you need.
+Lambda DB has a tiny codebase which does a lot
+Lambda is not ment as a persistance storage or a replacement for a traditional
+Database but as fast analytics engine cache representation engine.
 
-Can be used for your needs by changing the models.go file to your needs.
+powers: https://dego.vng.nl
+
+## Properties:
+
+- Insanely fast API. 1ms respsonses
+- Fast to setup.
+- Easy to deploy.
+- Easy to customize.
+- Easy export data
+
+- Implement custom authorized filters.
+
+## Indexes
+
+- S2 geoindex for fast point lookup
+- Bitarrays
+- Mapping
+
+- Your own special needs indexes!
+
+## Flow:
+
+Generate a model and load your data.
+The API is generated from your model.
+Deploy.
+
+Condition: Your dataset must fit in memory.
+
+Can be used for your needs by changing the `models.go` file to your needs.
 Creating and registering of the functionality that is needed.
 
 
 ### Steps
 You can start the database with only a csv.
 Go over steps below, And see the result in your browser.
+
 1. place csv file, in dir extras.
-2. `python3 create_model.py > ../model.go`
-3. cd ../
-4. go fmt
+2. `python3 create_model_.py`  answer the questions.
+3. go fmt model.go
+4. mv model.go ../
 5. go build
 6. ./lambda --help
 7. ./lambda  --csv assets/items.csv or `python3 ingestion.py -b 1000`
 9. curl 127.0.0.1:8128/help/
 10. browser 127.0.0.1:8128/
 
-
 11. instructions curl 127.0.0.1:8128/help/ | python -m json.tool
 
 
-
 ### Running
 
 sudo docker-compose up  --no-deps --build
 
 promql {instance="lambdadb:8000"}
 
 python3 extras/ingestion.py  -f movies_subset.tsv -format tsv -dbhost 127.0.0.1:8000
+=======
+
+1. instructions curl 127.0.0.1:8000/help/ | python -m json.tool
+
+### Questions
+
+
+
+### TODO
+
+- load data directly from a database (periodic)
+- document the `create_model.py` questions
+- use a remote source for CSV
+- use some compression faster to load than gzip
+- generate swagger API
+- Add more tests

csv.go

@@ -1,18 +1,17 @@
 package main
 
 import (
-	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
+	csv "github.com/JensRantil/go-csv"
+	"github.com/cheggaaa/pb"
+	"github.com/klauspost/pgzip"
 	"io"
 	"log"
 	"os"
 	"strings"
 	"unicode/utf8"
-
-	csv "github.com/JensRantil/go-csv"
-	"github.com/cheggaaa/pb"
 )
 
 func containsDelimiter(col string) bool {
@@ -59,11 +58,11 @@ func copyCSVRows(itemChan ItemsChannel, reader *csv.Reader, ignoreErrors bool,
 	success := 0
 	failed := 0
 
-	items := Items{}
+	items := ItemsIn{}
 
 	for {
-		item := Item{}
-		columns := item.Columns()
+		itemIn := ItemIn{}
+		columns := itemIn.Columns()
 		cols := make([]interface{}, len(columns))
 		record, err := reader.Read()
 
@@ -98,7 +97,7 @@ func copyCSVRows(itemChan ItemsChannel, reader *csv.Reader, ignoreErrors bool,
 		// marschall it to bytes
 		b, _ := json.Marshal(itemMap)
 		// fill the new Item instance with values
-		if err := json.Unmarshal([]byte(b), &item); err != nil {
+		if err := json.Unmarshal([]byte(b), &itemIn); err != nil {
 			line := strings.Join(record, delimiter)
 			failed++
 
@@ -113,14 +112,15 @@ func copyCSVRows(itemChan ItemsChannel, reader *csv.Reader, ignoreErrors bool,
 
 		if len(items) > 100000 {
 			itemChan <- items
-			items = Items{}
+			items = ItemsIn{}
 		}
-		items = append(items, &item)
+		items = append(items, &itemIn)
 		success++
 	}
 
 	// add leftover items
 	itemChan <- items
+	items = nil
 
 	return nil, success, failed
 }
@@ -142,7 +142,7 @@ func importCSV(filename string, itemChan ItemsChannel,
 		defer file.Close()
 
 		bar = NewProgressBar(file)
-		fz, err := gzip.NewReader(io.TeeReader(file, bar))
+		fz, err := pgzip.NewReader(io.TeeReader(file, bar))
 
 		if err != nil {
 			return err
@@ -179,7 +179,7 @@ func importCSV(filename string, itemChan ItemsChannel,
 		return fmt.Errorf("line %d: %s", lineNumber, err)
 	}
 
-	fmt.Printf("%d rows imported", success)
+	fmt.Printf("%d rows imported\n", success)
 
 	if ignoreErrors && failed > 0 {
 		fmt.Printf("%d rows could not be imported and have been written to stderr.", failed)

curlgeotest.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+set -u
+
+
+curl -vvv \
+	--data-urlencode 'geojson={
+				"type": "Polygon",
+				"coordinates": [
+					[
+					    [4.902321, 52.428306],
+					    [4.90127, 52.427024],
+					    [4.905281, 52.426069],
+					    [4.906782, 52.426226],
+					    [4.906418, 52.427469],
+					    [4.902321, 52.428306]
+					]
+				]
+			}' \
+	'http://127.0.0.1:8000/list/?groupby=postcode&reduce=count'

curltest.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -x
+set -e
+set -u
+
+# should be cached.
+curl -vv 'http://127.0.0.1:8000/list/?groupby=woning_type&reduce=count'
+
+# should not be cached.(using bitmaps)
+curl -vv 'http://127.0.0.1:8000/list/?match-wijkcode=WK036394&groupby=woning_type&reduce=count'

custom.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"strconv"
+)
+
+type registerCustomGroupByFunc map[string]func(*Item, ItemsGroupedBy)
+
+var RegisterGroupByCustom registerCustomGroupByFunc
+
+func init() {
+
+	RegisterGroupByCustom = make(registerCustomGroupByFunc)
+	RegisterGroupByCustom["gebruiksdoelen-mixed"] = GroupByGettersGebruiksdoelen
+
+}
+
+func reduceWEQ(items Items) map[string]string {
+	result := make(map[string]string)
+	weq := 0
+	for i := range items {
+		_weq, err := strconv.ParseInt(items[i].Woningequivalent, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+		weq += int(_weq)
+	}
+	result["woningenquivalent"] = strconv.Itoa(weq)
+	return result
+}