diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..92b8cfc --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,56 @@ +name: CI/CD + +on: + workflow_dispatch: + push: + branches: ["**"] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Setup Go + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - name: Build + run: | + mkdir -p bin + go build -o bin/pyrevit-telemetryserver + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: bin + path: bin + + - name: Run Tests + run: go test -v ./... + + docker: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub # TODO: uncomment if pushing + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build Docker image + run: docker build -t pyrevit-telemetryserver:latest . + + # - name: Push Docker image # Will be uncommented if pushing + # run: docker push pyrevit-telemetryserver:latest diff --git a/README.md b/README.md index 3e09386..00b2f5f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,56 @@ -# telemetry-server -Telemetry Server for pyRevit +# pyRevit Telemetry Server + +## Quick Start + +```sh +# Build the server +go build -o pyrevit-telemetryserver + +# Run with environment variables +PYREVT_TELEMETRY_DB_CONNSTRING="mongodb://localhost:27017/atoma-test" \ +PYREVT_TELEMETRY_SCRIPTS_TABLE="scripts" \ +PYREVT_TELEMETRY_EVENTS_TABLE="events" \ +./pyrevit-telemetryserver --port=8080 +``` + +## Environment Variables + +| Variable | Description | Example | +|----------------------------------------|---------------------------------------------|-----------------------------------------| +| PYREVT_TELEMETRY_DB_BACKEND | Database backend (`mongo`, `postgres`, etc) | mongo | +| PYREVT_TELEMETRY_DB_CONNSTRING | Database connection string | mongodb://localhost:27017/atoma-test | +| PYREVT_TELEMETRY_SCRIPTS_TABLE | Name of the scripts table/collection | scripts | +| PYREVT_TELEMETRY_EVENTS_TABLE | Name of the events table/collection | events | +| PYREVT_TELEMETRY_PORT | Port to run the server on | 8080 | +| PYREVT_TELEMETRY_DEBUG | Enable debug logging (`true`/`false`) | true | +| PYREVT_TELEMETRY_TRACE | Enable trace logging (`true`/`false`) | false | + +> All CLI options can be set via environment variables. CLI flags override environment variables. + +## API Endpoints + +- `GET /api/v1/status` — Health check +- `GET /api/v1/scripts/` — List v1 script telemetry +- `POST /api/v1/scripts/` — Submit v1 script telemetry +- `GET /api/v2/scripts/` — List v2 script telemetry +- `POST /api/v2/scripts/` — Submit v2 script telemetry +- `GET /api/v2/events/` — List v2 event telemetry +- `POST /api/v2/events/` — Submit v2 event telemetry + +## Example: Submit Script Telemetry + +```sh +curl -X POST -H "Content-Type: application/json" \ + -d '{"date":"2024-03-30","time":"08:45:00", ... }' \ + http://localhost:8080/api/v1/scripts/ +``` + +## Docker Usage + +### Build and Run with Docker Compose + +```sh +docker-compose up --build +``` + +This will start both DB and the telemetry server. The server will be available at `http://localhost:8080`. \ No newline at end of file diff --git a/cli/args.go b/cli/args.go new file mode 100644 index 0000000..dc9ff6d --- /dev/null +++ b/cli/args.go @@ -0,0 +1,121 @@ +package cli + +import ( + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/docopt/docopt-go" +) + +// Environment variable support: +// PYREVT_TELEMETRY_DB_CONNSTRING +// PYREVT_TELEMETRY_SCRIPTS_TABLE +// PYREVT_TELEMETRY_EVENTS_TABLE +// PYREVT_TELEMETRY_PORT +// PYREVT_TELEMETRY_HTTPS +// PYREVT_TELEMETRY_DEBUG +// PYREVT_TELEMETRY_TRACE +// PYREVT_TELEMETRY_EXENAME +// PYREVT_TELEMETRY_VERSION + +type Options struct { + ExeName string `json:"exe_name"` + Version string `json:"version"` + Opts *docopt.Opts + ConnString string `json:"connection_string"` + ScriptsTable string `json:"script_table"` + EventsTable string `json:"events_table"` + Port int `json:"server_port"` + Https bool `json:"https"` + Debug bool `json:"debug_mode"` + Trace bool `json:"trace_mode"` +} + +func getExeName() string { + return strings.TrimSuffix( + filepath.Base(os.Args[0]), + filepath.Ext(os.Args[0]), + ) +} + +func NewOptions() *Options { + argv := os.Args[1:] + + parser := &docopt.Parser{ + HelpHandler: printHelpAndExit, + } + + opts, _ := parser.ParseArgs(help, argv, version) + + connString, _ := opts.String("") + scriptTable, _ := opts.String("--scripts") + eventTable, _ := opts.String("--events") + port, _ := opts.Int("--port") + https, _ := opts.Bool("--https") + + debug, _ := opts.Bool("--debug") + trace, _ := opts.Bool("--trace") + + // Environment variable fallback + if connString == "" { + connString = os.Getenv("PYREVT_TELEMETRY_DB_CONNSTRING") + } + if scriptTable == "" { + scriptTable = os.Getenv("PYREVT_TELEMETRY_SCRIPTS_TABLE") + } + if eventTable == "" { + eventTable = os.Getenv("PYREVT_TELEMETRY_EVENTS_TABLE") + } + if port == 0 { + if portStr := os.Getenv("PYREVT_TELEMETRY_PORT"); portStr != "" { + if p, err := strconv.Atoi(portStr); err == nil { + port = p + } + } + } + if !https { + if httpsStr := os.Getenv("PYREVT_TELEMETRY_HTTPS"); httpsStr != "" { + if httpsStr == "1" || strings.ToLower(httpsStr) == "true" { + https = true + } + } + } + if !debug { + if debugStr := os.Getenv("PYREVT_TELEMETRY_DEBUG"); debugStr != "" { + if debugStr == "1" || strings.ToLower(debugStr) == "true" { + debug = true + } + } + } + if !trace { + if traceStr := os.Getenv("PYREVT_TELEMETRY_TRACE"); traceStr != "" { + if traceStr == "1" || strings.ToLower(traceStr) == "true" { + trace = true + } + } + } + + exeName := getExeName() + if envExe := os.Getenv("PYREVT_TELEMETRY_EXENAME"); envExe != "" { + exeName = envExe + } + ver := version + if envVer := os.Getenv("PYREVT_TELEMETRY_VERSION"); envVer != "" { + ver = envVer + } + + return &Options{ + ExeName: exeName, + Version: ver, + Opts: &opts, + ConnString: connString, + ScriptsTable: scriptTable, + EventsTable: eventTable, + Port: port, + Https: https, + Debug: debug, + Trace: trace, + } +} diff --git a/cli/logger.go b/cli/logger.go new file mode 100644 index 0000000..dfab80b --- /dev/null +++ b/cli/logger.go @@ -0,0 +1,39 @@ +package cli + +import ( + "log" + + "pkg.re/essentialkaos/ek.v10/fmtc" +) + +type Logger struct { + PrintDebug bool + PrintTrace bool +} + +func NewLogger(options *Options) *Logger { + return &Logger{ + PrintDebug: options.Debug, + PrintTrace: options.Trace, + } +} + +func (m *Logger) Fatal(args ...interface{}) { + log.Fatal(args...) +} + +func (m *Logger) Debug(args ...interface{}) { + if m.PrintDebug { + log.Print(args...) + } +} + +func (m *Logger) Trace(args ...interface{}) { + if m.PrintTrace { + log.Print(args...) + } +} + +func (m *Logger) Print(args ...interface{}) { + fmtc.Println(args...) +} diff --git a/cli/usage.go b/cli/usage.go new file mode 100644 index 0000000..d14b823 --- /dev/null +++ b/cli/usage.go @@ -0,0 +1,51 @@ +package cli + +import ( + "fmt" + "os" +) + +const version string = "0.19" +const help string = `Record pyRevit usage logs to database + +Usage: + pyrevit-telemetryserver [] [--scripts=] [--events=] --port= [--https] [--debug] [--trace] + +Options: + -h --help show this screen + -V --version show version + --scripts= target table or collection for script logs + --events= target table or collection for app event logs + --port= server port number to listen on + --https secure connection, expects ./pyrevit-telemetryserver.key and ./pyrevit-telemetryserver.crt + --debug print debug info + --trace print trace info e.g. full json logs and sql queries + +Supports: + postgresql: using github.com/lib/pq + mongodb: using gopkg.in/mgo.v2 + mysql: using github.com/go-sql-driver/mysql + sqlserver: using github.com/denisenkom/go-mssqldb + sqlite3: using github.com/mattn/go-sqlite3 + +Examples: + pyrevit-telemetryserver postgres://user:pass@data.mycompany.com/mydb --scripts="pyrevitlogs" --events="appevents" --port=8080 --debug + pyrevit-telemetryserver mongodb://user:pass@localhost:27017/mydb --scripts="pyrevitlogs" --events="appevents" --port=8080 + pyrevit-telemetryserver "mysql:user:pass@tcp(localhost:3306)/tests" --scripts="pyrevitlogs" --port=8080 + pyrevit-telemetryserver sqlserver://user:pass@my-azure-db.database.windows.net?database=mydb --scripts="pyrevitlogs" --port=8080 + pyrevit-telemetryserver sqlite3:data.db --scripts="pyrevitlogs" --port=8080 +` + +var printHelpAndExit = func(err error, docoptMessage string) { + if err != nil { + // if err occured print full help + // docopt only includes usage section in its message + fmt.Fprint(os.Stderr, help) + os.Exit(1) + } else { + // otherwise print whatever docopt says + // e.g. reporting version + fmt.Println(docoptMessage) + os.Exit(0) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..48d7f4a --- /dev/null +++ b/go.mod @@ -0,0 +1,40 @@ +module pyrevittelemetryserver + +go 1.23.0 + +toolchain go1.24.2 + +require ( + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d + github.com/denisenkom/go-mssqldb v0.11.0 + github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 + github.com/go-sql-driver/mysql v1.6.0 + github.com/gofrs/uuid v4.3.1+incompatible + github.com/gorilla/mux v1.8.0 + github.com/lib/pq v1.10.3 + github.com/mattn/go-sqlite3 v1.14.8 + github.com/pkg/errors v0.9.1 + github.com/satori/go.uuid v1.2.0 + go.mongodb.org/mongo-driver v1.11.1 + pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible +) + +require github.com/stretchr/testify v1.10.0 // indirect + +require ( + // Indirect dependencies + github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.1 // indirect + github.com/xdg-go/stringprep v1.0.3 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/text v0.22.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + pkg.re/essentialkaos/check.v1 v1.0.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..092d64b --- /dev/null +++ b/go.sum @@ -0,0 +1,88 @@ +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= +github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= +github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= +go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +pkg.re/essentialkaos/check.v1 v1.0.0 h1:2V++mhtm9yHqvW7gtXqcU1D+98vTICGnXmaZloLsZVY= +pkg.re/essentialkaos/check.v1 v1.0.0/go.mod h1:B7CoMnGFRnruw7X2Z45kWNvoCW+5OhUsLUm1EBM1aJs= +pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible h1:MSnAZgf9WxV/kBpmPpD7md3ajOSXrugvbGIqRd9AWTI= +pkg.re/essentialkaos/ek.v10 v12.32.0+incompatible/go.mod h1:QhFbmORYfukHQjR05vj21bPWmCRLYlSy0tNGGCQgGnI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..c32e553 --- /dev/null +++ b/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + "pyrevittelemetryserver/server" +) + +func main() { + // process command line arguments + options := cli.NewOptions() + + // Then log options if requested + logger := cli.NewLogger(options) + logger.Trace(options) + for key, value := range *options.Opts { + logger.Debug(fmt.Sprintf("%s=%v", key, value)) + } + + dbcfg, cErr := persistence.NewConfig(options) + if cErr != nil { + panic(cErr) + } + + // request a db connection to read and write + dbConn, nErr := persistence.NewConnection(dbcfg) + if nErr != nil { + panic(nErr) + } + + // ask server to start and pass db writer interface + server.Start(options, dbConn, logger) +} diff --git a/persistence/config.go b/persistence/config.go new file mode 100644 index 0000000..6ef8609 --- /dev/null +++ b/persistence/config.go @@ -0,0 +1,56 @@ +package persistence + +import ( + "strings" + + "pyrevittelemetryserver/cli" + + "github.com/pkg/errors" +) + +type DBBackend string + +const ( + Postgres DBBackend = "postgres" + MongoDB DBBackend = "mongodb" + MySql DBBackend = "mysql" + MSSql DBBackend = "sqlserver" + Sqlite DBBackend = "sqlite3" +) + +type Config struct { + Backend DBBackend `json:"backend"` + ConnString string `json:"connection_string"` + ScriptTarget string `json:"script_target"` + EventTarget string `json:"event_target"` +} + +func NewConfig(options *cli.Options) (*Config, error) { + backend, err := parseUri(options.ConnString) + if err != nil { + return nil, err + } + + return &Config{ + Backend: backend, + ConnString: options.ConnString, + ScriptTarget: options.ScriptsTable, + EventTarget: options.EventsTable, + }, nil +} + +func parseUri(connString string) (DBBackend, error) { + if strings.HasPrefix(connString, "postgres:") { + return Postgres, nil + } else if strings.HasPrefix(connString, "mongodb:") { + return MongoDB, nil + } else if strings.HasPrefix(connString, "mysql:") { + return MySql, nil + } else if strings.HasPrefix(connString, "sqlserver:") { + return MSSql, nil + } else if strings.HasPrefix(connString, "sqlite3:") { + return Sqlite, nil + } else { + return "", errors.New("db is not yet supported") + } +} diff --git a/persistence/connection.go b/persistence/connection.go new file mode 100644 index 0000000..6f67240 --- /dev/null +++ b/persistence/connection.go @@ -0,0 +1,64 @@ +package persistence + +import ( + "pyrevittelemetryserver/cli" +) + +type ConnectionStatus struct { + Status string `json:"status"` + Version string `json:"version"` + Output string `json:"output"` +} + +// ErroCodes +// 0: All OK +// 1: No data to write +// 2: data is available but did not get pushed under dry run +// 3: headers are required +type Result struct { + ResultCode int + Message string +} + +type DatabaseConnection struct { + Config *Config `json:"db_configs"` +} + +type Connection interface { + GetType() DBBackend + GetVersion(*cli.Logger) string + GetStatus(*cli.Logger) ConnectionStatus + WriteScriptTelemetryV1(*ScriptTelemetryRecordV1, *cli.Logger) (*Result, error) + WriteScriptTelemetryV2(*ScriptTelemetryRecordV2, *cli.Logger) (*Result, error) + WriteEventTelemetryV2(*EventTelemetryRecordV2, *cli.Logger) (*Result, error) + + // Read methods for retrieving telemetry data + ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) + ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) + ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) + + // Search methods for filtering telemetry data + SearchScriptTelemetryV1(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) + SearchScriptTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) + SearchEventTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) +} + +func NewConnection(dbcfg *Config) (Connection, error) { + w := DatabaseConnection{ + Config: dbcfg, + } + if dbcfg.Backend == Postgres { + return GenericSQLConnection{w}, nil + } else if dbcfg.Backend == MongoDB { + return MongoDBConnection{w}, nil + } else if dbcfg.Backend == MySql { + return GenericSQLConnection{w}, nil + } else if dbcfg.Backend == MSSql { + return GenericSQLConnection{w}, nil + } else if dbcfg.Backend == Sqlite { + return GenericSQLConnection{w}, nil + } + // ... other writers + + panic("should not get here") +} diff --git a/persistence/genericsql.go b/persistence/genericsql.go new file mode 100644 index 0000000..579eb51 --- /dev/null +++ b/persistence/genericsql.go @@ -0,0 +1,414 @@ +package persistence + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + "reflect" + "regexp" + "strconv" + "strings" + + "pyrevittelemetryserver/cli" + + _ "github.com/denisenkom/go-mssqldb" + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + _ "github.com/mattn/go-sqlite3" + + uuid "github.com/satori/go.uuid" +) + +type GenericSQLConnection struct { + DatabaseConnection +} + +func (w GenericSQLConnection) GetType() DBBackend { + return w.Config.Backend +} + +func (w GenericSQLConnection) GetVersion(logger *cli.Logger) string { + db, err := openConnection(w.Config.Backend, w.Config.ConnString, logger) + if err != nil { + logger.Debug("error opening connection") + return "" + } + defer db.Close() + + var version string + err = db.QueryRow("select version()").Scan(&version) + if err != nil { + err = db.QueryRow("select @@version").Scan(&version) + if err != nil { + log.Fatal(err) + } + } + return version +} + +func (w GenericSQLConnection) GetStatus(logger *cli.Logger) ConnectionStatus { + return ConnectionStatus{ + Status: "pass", + Version: w.GetVersion(logger), + } +} + +func (w GenericSQLConnection) WriteScriptTelemetryV1(logrec *ScriptTelemetryRecordV1, logger *cli.Logger) (*Result, error) { + // generate generic sql insert query + logger.Debug("generating query") + query, qErr := generateScriptInsertQueryV1(w.Config.ScriptTarget, logrec, logger) + if qErr != nil { + return nil, qErr + } + + return commitSQL(w.Config.Backend, w.Config.ConnString, query, logger) +} + +func (w GenericSQLConnection) WriteScriptTelemetryV2(logrec *ScriptTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + // generate generic sql insert query + logger.Debug("generating query") + query, qErr := generateScriptInsertQueryV2(w.Config.ScriptTarget, logrec, logger) + if qErr != nil { + return nil, qErr + } + + return commitSQL(w.Config.Backend, w.Config.ConnString, query, logger) +} + +func (w GenericSQLConnection) WriteEventTelemetryV2(logrec *EventTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + // generate generic sql insert query + logger.Debug("generating query") + query, qErr := generateEventInsertQueryV2(w.Config.EventTarget, logrec, logger) + if qErr != nil { + return nil, qErr + } + + return commitSQL(w.Config.Backend, w.Config.ConnString, query, logger) +} + +func commitSQL(backend DBBackend, connStr string, query string, logger *cli.Logger) (*Result, error) { + // open connection + db, err := openConnection(backend, connStr, logger) + if err != nil { + logger.Debug("error opening connection") + return nil, err + } + defer db.Close() + + // start transaction + logger.Debug("opening transaction") + tx, beginErr := db.Begin() + if beginErr != nil { + logger.Debug("error opening transaction") + return nil, beginErr + } + defer tx.Rollback() + + // run the insert query + logger.Debug("executing insert query") + _, eErr := db.Exec(query) + if eErr != nil { + return nil, eErr + } + + // commit transaction + logger.Debug("commiting transaction") + txnErr := tx.Commit() + if txnErr != nil { + return nil, txnErr + } + + logger.Debug("preparing report") + return &Result{ + Message: "successfully inserted usage record", + }, nil +} + +func openConnection(backend DBBackend, connStr string, logger *cli.Logger) (*sql.DB, error) { + // open connection + logger.Debug(fmt.Sprintf("opening %s connection", backend)) + cleanConnStr := connStr + if backend == Sqlite || backend == MySql { + cleanConnStr = strings.Replace(connStr, string(backend)+":", "", 1) + } + return sql.Open(string(backend), cleanConnStr) +} + +func generateScriptInsertQueryV1(table string, logrec *ScriptTelemetryRecordV1, logger *cli.Logger) (string, error) { + // read csv file and build sql insert query + var querystr strings.Builder + + logger.Debug("generating insert query with-out headers") + querystr.WriteString(fmt.Sprintf("INSERT INTO %s values ", table)) + + // build sql data info + logger.Debug("building insert query for data") + datalines := make([]string, 0) + + cresults, merr := json.Marshal(logrec.CommandResults) + if merr != nil { + logger.Debug("error logging command results") + } + + // create record based on schema + var record []string + + // generate record id, panic if error + recordId := uuid.NewV4() + + re := regexp.MustCompile(`(\d+:\d+:\d+)`) + record = []string{ + recordId.String(), + logrec.Date, + re.FindString(logrec.Time), + logrec.UserName, + logrec.RevitVersion, + logrec.RevitBuild, + logrec.SessionId, + logrec.PyRevitVersion, + strconv.FormatBool(logrec.IsDebugMode), + strconv.FormatBool(logrec.IsConfigMode), + logrec.CommandName, + logrec.BundleName, + logrec.ExtensionName, + logrec.CommandUniqueName, + strconv.Itoa(logrec.ResultCode), + string(cresults), + logrec.ScriptPath, + logrec.TraceInfo.EngineInfo.Version, + logrec.TraceInfo.IronPythonTraceDump, + logrec.TraceInfo.CLRTraceDump, + } + + datalines = append(datalines, ToSql(&record, true)) + + // add csv records to query string + all_datalines := strings.Join(datalines, ", ") + logger.Trace(all_datalines) + querystr.WriteString(all_datalines) + querystr.WriteString(";\n") + logger.Debug("building query completed") + + // execute query + full_query := querystr.String() + logger.Trace(full_query) + return full_query, nil +} + +func generateScriptInsertQueryV2(table string, logrec *ScriptTelemetryRecordV2, logger *cli.Logger) (string, error) { + // read csv file and build sql insert query + var querystr strings.Builder + + logger.Debug("generating insert query with-out headers") + querystr.WriteString(fmt.Sprintf("INSERT INTO %s values ", table)) + + // build sql data info + logger.Debug("building insert query for data") + datalines := make([]string, 0) + + // marshal json data + engineCfgs, merr := json.Marshal(logrec.TraceInfo.EngineInfo.Configs) + if merr != nil { + logger.Debug("error logging engine configs") + } + + // marshal json data + cresults, merr := json.Marshal(logrec.CommandResults) + if merr != nil { + logger.Debug("error logging command results") + } + + // create record based on schema + var record []string + + // generate record id, panic if error + recordId := uuid.NewV4() + + record = []string{ + recordId.String(), + logrec.TimeStamp, + logrec.UserName, + logrec.HostUserName, + logrec.RevitVersion, + logrec.RevitBuild, + logrec.SessionId, + logrec.PyRevitVersion, + logrec.Clone, + strconv.FormatBool(logrec.IsDebugMode), + strconv.FormatBool(logrec.IsConfigMode), + strconv.FormatBool(logrec.IsExecFromGUI), + logrec.ExecId, + logrec.ExecTimeStamp, + logrec.CommandName, + logrec.BundleName, + logrec.ExtensionName, + logrec.CommandUniqueName, + logrec.DocumentName, + logrec.DocumentPath, + strconv.Itoa(logrec.ResultCode), + string(cresults), + logrec.ScriptPath, + logrec.TraceInfo.EngineInfo.Type, + logrec.TraceInfo.EngineInfo.Version, + strings.Join(logrec.TraceInfo.EngineInfo.SysPaths, ";"), + string(engineCfgs), + logrec.TraceInfo.Message, + } + datalines = append(datalines, ToSql(&record, true)) + + // add csv records to query string + all_datalines := strings.Join(datalines, ", ") + logger.Trace(all_datalines) + querystr.WriteString(all_datalines) + querystr.WriteString(";\n") + logger.Debug("building query completed") + + // execute query + full_query := querystr.String() + logger.Trace(full_query) + return full_query, nil +} + +func generateEventInsertQueryV2(table string, logrec *EventTelemetryRecordV2, logger *cli.Logger) (string, error) { + // read csv file and build sql insert query + var querystr strings.Builder + + logger.Debug("generating insert query with-out headers") + querystr.WriteString(fmt.Sprintf("INSERT INTO %s values ", table)) + + // build sql data info + logger.Debug("building insert query for data") + datalines := make([]string, 0) + + // marshal json data + cresults, merr := json.Marshal(logrec.EventArgs) + if merr != nil { + logger.Debug("error logging command results") + } + + // create record based on schema + var record []string + + // generate record id, panic if error + recordId := uuid.NewV4() + + record = []string{ + recordId.String(), + logrec.TimeStamp, + logrec.HandlerId, + logrec.EventType, + string(cresults), + logrec.UserName, + logrec.HostUserName, + logrec.RevitVersion, + logrec.RevitBuild, + strconv.FormatBool(logrec.Cancellable), + strconv.FormatBool(logrec.Cancelled), + strconv.Itoa(logrec.DocumentId), + logrec.DocumentType, + logrec.DocumentTemplate, + logrec.DocumentName, + logrec.DocumentPath, + logrec.ProjectNumber, + logrec.ProjectName, + } + datalines = append(datalines, ToSql(&record, true)) + + // add csv records to query string + all_datalines := strings.Join(datalines, ", ") + logger.Trace(all_datalines) + querystr.WriteString(all_datalines) + querystr.WriteString(";\n") + logger.Debug("building query completed") + + // execute query + full_query := querystr.String() + logger.Trace(full_query) + return full_query, nil +} + +// Read methods for SQL databases +func (w GenericSQLConnection) ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readSQL[ScriptTelemetryRecordV1](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w GenericSQLConnection) ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readSQL[ScriptTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w GenericSQLConnection) ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readSQL[EventTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.EventTarget, nil, limit, offset, logger) +} + +// Search methods for SQL databases +func (w GenericSQLConnection) SearchScriptTelemetryV1(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readSQL[ScriptTelemetryRecordV1](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w GenericSQLConnection) SearchScriptTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readSQL[ScriptTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w GenericSQLConnection) SearchEventTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readSQL[EventTelemetryRecordV2](w.Config.Backend, w.Config.ConnString, w.Config.EventTarget, query, limit, offset, logger) +} + +// Generic read function for SQL databases +func readSQL[T any](backend DBBackend, connStr string, table string, query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]T, error) { + // Open connection + db, err := openConnection(backend, connStr, logger) + if err != nil { + return nil, err + } + defer db.Close() + + // Build query + var sqlQuery strings.Builder + sqlQuery.WriteString(fmt.Sprintf("SELECT * FROM %s", table)) + + // Add WHERE clause if query parameters provided + args := make([]interface{}, 0) + if query != nil { + sqlQuery.WriteString(" WHERE ") + conditions := make([]string, 0) + for key, value := range query { + conditions = append(conditions, fmt.Sprintf("%s = ?", key)) + args = append(args, value) + } + sqlQuery.WriteString(strings.Join(conditions, " AND ")) + } + + // Add ORDER BY and LIMIT/OFFSET + sqlQuery.WriteString(" ORDER BY timestamp DESC") + sqlQuery.WriteString(fmt.Sprintf(" LIMIT %d OFFSET %d", limit, offset)) + + // Execute query + rows, err := db.Query(sqlQuery.String(), args...) + if err != nil { + return nil, err + } + defer rows.Close() + + // Scan results + var results []T + for rows.Next() { + var record T + // Use reflection to scan into struct fields + val := reflect.ValueOf(&record).Elem() + fields := make([]interface{}, val.NumField()) + for i := 0; i < val.NumField(); i++ { + fields[i] = val.Field(i).Addr().Interface() + } + if err := rows.Scan(fields...); err != nil { + return nil, err + } + results = append(results, record) + } + + if err = rows.Err(); err != nil { + return nil, err + } + + return results, nil +} diff --git a/persistence/models.go b/persistence/models.go new file mode 100644 index 0000000..0cc6a0c --- /dev/null +++ b/persistence/models.go @@ -0,0 +1,208 @@ +package persistence + +import ( + "fmt" + + "pyrevittelemetryserver/cli" + + "github.com/asaskevich/govalidator" +) + +// v1.0 +type EngineInfoV1 struct { + Version string `json:"version" bson:"version" valid:"-"` + SysPaths []string `json:"syspath" bson:"syspath" valid:"-"` +} + +type TraceInfoV1 struct { + EngineInfo EngineInfoV1 `json:"engine" bson:"engine" valid:"-"` + IronPythonTraceDump string `json:"ipy" bson:"ipy" valid:"-"` + CLRTraceDump string `json:"clr" bson:"clr" valid:"-"` +} + +type ScriptTelemetryRecordV1 struct { + Date string `json:"date" bson:"date" valid:"-"` + Time string `json:"time" bson:"time" valid:"-"` + UserName string `json:"username" bson:"username" valid:"-"` + RevitVersion string `json:"revit" bson:"revit" valid:"numeric~Invalid revit version"` + RevitBuild string `json:"revitbuild" bson:"revitbuild" valid:"matches(\\d{8}_\\d{4}\\(x\\d{2}\\))~Invalid revit build number"` + SessionId string `json:"sessionid" bson:"sessionid" valid:"uuidv4~Invalid session id"` + PyRevitVersion string `json:"pyrevit" bson:"pyrevit" valid:"-"` + IsDebugMode bool `json:"debug" bson:"debug"` + IsConfigMode bool `json:"config" bson:"config"` + CommandName string `json:"commandname" bson:"commandname" valid:"-"` + CommandUniqueName string `json:"commanduniquename" bson:"commanduniquename" valid:"-"` + BundleName string `json:"commandbundle" bson:"commandbundle" valid:"-"` + ExtensionName string `json:"commandextension" bson:"commandextension" valid:"-"` + ResultCode int `json:"resultcode" bson:"resultcode" valid:"numeric~Invalid result code"` + CommandResults map[string]string `json:"commandresults" bson:"commandresults" valid:"-"` + ScriptPath string `json:"scriptpath" bson:"scriptpath" valid:"-"` + TraceInfo TraceInfoV1 `json:"trace" bson:"trace"` +} + +func (logrec ScriptTelemetryRecordV1) PrintRecordInfo(logger *cli.Logger, message string) { + logger.Print(fmt.Sprintf( + "%s %s-%s %q @ %s:%s [%s.%s] code=%d info=%v", + message, + logrec.Date, + logrec.Time, + logrec.UserName, + logrec.RevitBuild, + logrec.TraceInfo.EngineInfo.Version, + logrec.ExtensionName, + logrec.CommandName, + logrec.ResultCode, + logrec.CommandResults, + )) +} + +func (logrec ScriptTelemetryRecordV1) Validate() error { + // govalidator.SetFieldsRequiredByDefault(true) + + // validate now + _, err := govalidator.ValidateStruct(logrec) + return err +} + +// v2.0 +type EngineInfoV2 struct { + Type string `json:"type" bson:"type" valid:"engine~Invalid executor engine type"` + Version string `json:"version" bson:"version" valid:"-"` + SysPaths []string `json:"syspath" bson:"syspath" valid:"-"` + Configs map[string]interface{} `json:"configs" bson:"configs" valid:"-"` +} + +type TraceInfoV2 struct { + EngineInfo EngineInfoV2 `json:"engine" bson:"engine"` + Message string `json:"message" bson:"message" valid:"-"` +} + +type RecordMetaV2 struct { + SchemaVersion string `json:"schema" bson:"schema" valid:"schema~Invalid schema version"` +} + +type ScriptTelemetryRecordV2 struct { + RecordMeta RecordMetaV2 `json:"meta" bson:"meta"` + TimeStamp string `json:"timestamp" bson:"timestamp" valid:"rfc3339~Invalid timestamp"` + UserName string `json:"username" bson:"username" valid:"-"` + HostUserName string `json:"host_user" bson:"host_user" valid:"-"` + RevitVersion string `json:"revit" bson:"revit" valid:"numeric~Invalid revit version"` + RevitBuild string `json:"revitbuild" bson:"revitbuild" valid:"matches(\\d{8}_\\d{4}\\(x\\d{2}\\))~Invalid revit build number"` + SessionId string `json:"sessionid" bson:"sessionid" valid:"uuidv4~Invalid session id"` + PyRevitVersion string `json:"pyrevit" bson:"pyrevit" valid:"-"` + Clone string `json:"clone" bson:"clone" valid:"-"` + IsDebugMode bool `json:"debug" bson:"debug"` + IsConfigMode bool `json:"config" bson:"config"` + IsExecFromGUI bool `json:"from_gui" bson:"from_gui"` + ExecId string `json:"exec_id" bson:"exec_id" valid:"-"` + ExecTimeStamp string `json:"exec_timestamp" bson:"exec_timestamp" valid:"-"` + CommandName string `json:"commandname" bson:"commandname" valid:"-"` + CommandUniqueName string `json:"commanduniquename" bson:"commanduniquename" valid:"-"` + BundleName string `json:"commandbundle" bson:"commandbundle" valid:"-"` + ExtensionName string `json:"commandextension" bson:"commandextension" valid:"-"` + DocumentName string `json:"docname" bson:"docname" valid:"-"` + DocumentPath string `json:"docpath" bson:"docpath" valid:"-"` + ResultCode int `json:"resultcode" bson:"resultcode" valid:"numeric~Invalid result code"` + CommandResults map[string]interface{} `json:"commandresults" bson:"commandresults" valid:"-"` + ScriptPath string `json:"scriptpath" bson:"scriptpath" valid:"-"` + TraceInfo TraceInfoV2 `json:"trace" bson:"trace"` +} + +func (logrec ScriptTelemetryRecordV2) PrintRecordInfo(logger *cli.Logger, message string) { + logger.Print(fmt.Sprintf( + "%s %s %q %s:%s (%s) [%s.%s] code=%d info=%v", + message, + logrec.TimeStamp, + logrec.UserName, + logrec.RevitBuild, + logrec.TraceInfo.EngineInfo.Version, + logrec.TraceInfo.EngineInfo.Type, + logrec.ExtensionName, + logrec.CommandName, + logrec.ResultCode, + logrec.CommandResults, + )) +} + +func (logrec ScriptTelemetryRecordV2) Validate() error { + // govalidator.SetFieldsRequiredByDefault(true) + + // custom validators + govalidator.TagMap["schema"] = govalidator.Validator(func(str string) bool { + return str == "2.0" + }) + + govalidator.TagMap["engine"] = govalidator.Validator(func(str string) bool { + switch str { + case + "unknown", + "ironpython", + "cpython", + "csharp", + "invoke", + "visualbasic", + "ironruby", + "dynamobim", + "grasshopper", + "content", + "hyperlink": + return true + } + return false + }) + + // validate now + _, err := govalidator.ValidateStruct(logrec) + return err +} + +// introduced with api v2 +type EventTelemetryRecordV2 struct { + RecordMeta RecordMetaV2 `json:"meta" bson:"meta"` + TimeStamp string `json:"timestamp" bson:"timestamp" valid:"rfc3339~Invalid timestamp"` + HandlerId string `json:"handler_id" bson:"handler_id" valid:"-"` + EventType string `json:"type" bson:"type" valid:"-"` + EventArgs map[string]interface{} `json:"args" bson:"args" valid:"-"` + UserName string `json:"username" bson:"username" valid:"-"` + HostUserName string `json:"host_user" bson:"host_user" valid:"-"` + RevitVersion string `json:"revit" bson:"revit" valid:"numeric~Invalid revit version"` + RevitBuild string `json:"revitbuild" bson:"revitbuild" valid:"matches(\\d{8}_\\d{4}\\(x\\d{2}\\))~Invalid revit build number"` + + // general + Cancellable bool `json:"cancellable" bson:"cancellable"` + Cancelled bool `json:"cancelled" bson:"cancelled"` + DocumentId int `json:"docid" bson:"docid" valid:"-"` + DocumentType string `json:"doctype" bson:"doctype" valid:"-"` + DocumentTemplate string `json:"doctemplate" bson:"doctemplate" valid:"-"` + DocumentName string `json:"docname" bson:"docname" valid:"-"` + DocumentPath string `json:"docpath" bson:"docpath" valid:"-"` + ProjectNumber string `json:"projectnum" bson:"projectnum" valid:"-"` + ProjectName string `json:"projectname" bson:"projectname" valid:"-"` +} + +func (logrec EventTelemetryRecordV2) PrintRecordInfo(logger *cli.Logger, message string) { + if logrec.RecordMeta.SchemaVersion == "2.0" { + logger.Print(fmt.Sprintf( + "%s %s [%s] %q @ %s doc=%q @ %s", + message, + logrec.TimeStamp, + logrec.EventType, + logrec.HostUserName, + logrec.RevitBuild, + logrec.DocumentName, + logrec.DocumentPath, + )) + } +} + +func (logrec EventTelemetryRecordV2) Validate() error { + // govalidator.SetFieldsRequiredByDefault(true) + + // custom validators + govalidator.TagMap["schema"] = govalidator.Validator(func(str string) bool { + return str == "2.0" + }) + + _, err := govalidator.ValidateStruct(logrec) + return err +} diff --git a/persistence/mongo.go b/persistence/mongo.go new file mode 100644 index 0000000..16910ba --- /dev/null +++ b/persistence/mongo.go @@ -0,0 +1,209 @@ +package persistence + +import ( + "context" + "fmt" + "pyrevittelemetryserver/cli" + "time" + + _ "github.com/lib/pq" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" +) + +type MongoDBConnection struct { + DatabaseConnection +} + +func (w MongoDBConnection) GetType() DBBackend { + return w.Config.Backend +} + +func (w MongoDBConnection) GetVersion(logger *cli.Logger) string { + // parse and grab database name from uri + logger.Debug("grabbing db name from connection string") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + logger.Debug("opening mongodb session") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(w.Config.ConnString)) + + defer func() { + if err = client.Disconnect(ctx); err != nil { + panic(err) + } + }() + + ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + pErr := client.Ping(ctx, readpref.Primary()) + + if pErr != nil { + return "" + } + + // get version from admin DB + logger.Debug("getting mongodb version") + var commandResult bson.M + command := bson.D{{"buildInfo", 1}} + vErr := client.Database("admin").RunCommand(ctx, command).Decode(&commandResult) + + if vErr != nil { + return "" + } + + // parse version field to get version information + ver := fmt.Sprintf("%+v", commandResult["version"]) + return ver +} + +func (w MongoDBConnection) GetStatus(logger *cli.Logger) ConnectionStatus { + return ConnectionStatus{ + Status: "pass", + Version: w.GetVersion(logger), + } +} + +func (w MongoDBConnection) WriteScriptTelemetryV1(logrec *ScriptTelemetryRecordV1, logger *cli.Logger) (*Result, error) { + return commitMongo(w.Config.ConnString, w.Config.ScriptTarget, logrec, logger) +} + +func (w MongoDBConnection) WriteScriptTelemetryV2(logrec *ScriptTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + return commitMongo(w.Config.ConnString, w.Config.ScriptTarget, logrec, logger) +} + +func (w MongoDBConnection) WriteEventTelemetryV2(logrec *EventTelemetryRecordV2, logger *cli.Logger) (*Result, error) { + return commitMongo(w.Config.ConnString, w.Config.EventTarget, logrec, logger) +} + +func commitMongo(connStr string, targetCollection string, logrec interface{}, logger *cli.Logger) (*Result, error) { + // parse and grab database name from uri + logger.Debug("check connection string") + connStringInfo, err := connstring.ParseAndValidate(connStr) + + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + logger.Debug("opening mongodb session using connection string") + client, err := mongo.Connect(ctx, options.Client().ApplyURI(connStr)) + + if err != nil { + return nil, err + } + + logger.Trace(client) + + logger.Debug("getting target collection") + // db := session.DB(dialinfo.Database) + db := client.Database(connStringInfo.Database) + // c := db.C(targetCollection) + c := db.Collection(targetCollection) + logger.Trace(c) + + logger.Debug("opening bulk operation") + // bulkop := c.Bulk() + + // build sql data info + logger.Debug("building documents") + + iCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + logger.Debug("inserting new document") + _, txnErr := c.InsertOne(iCtx, logrec) + + if txnErr != nil { + return nil, txnErr + } + + // compact collection if requested + logger.Debug("preparing report") + return &Result{ + Message: "successfully inserted usage document", + }, nil +} + +// Read methods for MongoDB +func (w MongoDBConnection) ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readMongo[ScriptTelemetryRecordV1](w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w MongoDBConnection) ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readMongo[ScriptTelemetryRecordV2](w.Config.ConnString, w.Config.ScriptTarget, nil, limit, offset, logger) +} + +func (w MongoDBConnection) ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readMongo[EventTelemetryRecordV2](w.Config.ConnString, w.Config.EventTarget, nil, limit, offset, logger) +} + +// Search methods for MongoDB +func (w MongoDBConnection) SearchScriptTelemetryV1(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV1, error) { + return readMongo[ScriptTelemetryRecordV1](w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w MongoDBConnection) SearchScriptTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]ScriptTelemetryRecordV2, error) { + return readMongo[ScriptTelemetryRecordV2](w.Config.ConnString, w.Config.ScriptTarget, query, limit, offset, logger) +} + +func (w MongoDBConnection) SearchEventTelemetryV2(query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]EventTelemetryRecordV2, error) { + return readMongo[EventTelemetryRecordV2](w.Config.ConnString, w.Config.EventTarget, query, limit, offset, logger) +} + +// Generic read function for MongoDB +func readMongo[T any](connStr string, targetCollection string, query map[string]interface{}, limit int, offset int, logger *cli.Logger) ([]T, error) { + // Parse connection string + connStringInfo, err := connstring.ParseAndValidate(connStr) + if err != nil { + return nil, err + } + + // Connect to MongoDB + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + client, err := mongo.Connect(ctx, options.Client().ApplyURI(connStr)) + if err != nil { + return nil, err + } + defer client.Disconnect(ctx) + + // Get collection + db := client.Database(connStringInfo.Database) + c := db.Collection(targetCollection) + + // Prepare query options + findOptions := options.Find() + findOptions.SetLimit(int64(limit)) + findOptions.SetSkip(int64(offset)) + findOptions.SetSort(bson.D{{"timestamp", -1}}) // Sort by timestamp descending + + // Convert query map to bson.M + var filter bson.M + if query != nil { + filter = bson.M(query) + } else { + filter = bson.M{} + } + + // Execute query + cursor, err := c.Find(ctx, filter, findOptions) + if err != nil { + return nil, err + } + defer cursor.Close(ctx) + + // Decode results + var results []T + if err = cursor.All(ctx, &results); err != nil { + return nil, err + } + + return results, nil +} diff --git a/persistence/utils.go b/persistence/utils.go new file mode 100644 index 0000000..d460268 --- /dev/null +++ b/persistence/utils.go @@ -0,0 +1,35 @@ +package persistence + +import ( + "fmt" + "strings" +) + +func ToSql(values *[]string, wrap bool) string { + // wrap values in '' first + cleanedValues := make([]string, 0) + valueFormat := "%s" + if wrap { + for _, value := range *values { + if value != "" { + cleanedValues = append( + cleanedValues, + fmt.Sprintf("'%s'", strings.Replace(value, "'", "''", -1))) + } else { + cleanedValues = append(cleanedValues, "NULL") + } + } + } else { + for _, value := range *values { + cleanedValues = append( + cleanedValues, + fmt.Sprintf(valueFormat, value)) + } + } + // create the (,,,) sql value list + return fmt.Sprintf("(%s)", strings.Join(cleanedValues, ", ")) +} + +func ToMap(fields, values *[]string) map[string]string { + return make(map[string]string) +} diff --git a/server/common.go b/server/common.go new file mode 100644 index 0000000..dc90d32 --- /dev/null +++ b/server/common.go @@ -0,0 +1,18 @@ +package server + +import ( + "net/http" + + "pyrevittelemetryserver/cli" +) + +const OkMessage = "[ {g}OK{!} ]" + +func respondError(err error, w http.ResponseWriter, logger *cli.Logger) { + message := err.Error() + logger.Debug("validation error: ", message) + _, responseErr := w.Write([]byte(message)) + if responseErr != nil { + logger.Debug(responseErr) + } +} diff --git a/server/events.go b/server/events.go new file mode 100644 index 0000000..fd47568 --- /dev/null +++ b/server/events.go @@ -0,0 +1,124 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gorilla/mux" +) + +func dumpEventAndRespond(logrec interface{}, w http.ResponseWriter, logger *cli.Logger) { + jsonData, responseDataErr := json.Marshal(logrec) + if responseDataErr == nil { + jsonString := string(jsonData) + if logger.PrintTrace { + logger.Trace(jsonString) + } + w.Header().Set("Content-Type", "application/json") + _, responseErr := w.Write([]byte(jsonString)) + if responseErr != nil { + logger.Debug(responseErr) + } + } else { + logger.Debug(responseDataErr) + } +} + +func RouteEvents(router *mux.Router, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // POST events/ + // create new script telemetry record + // https://stackoverflow.com/a/26212073 + router.HandleFunc("/api/v2/events/", func(w http.ResponseWriter, r *http.Request) { + // parse given json data into a new record + logrec := persistence.EventTelemetryRecordV2{} + decodeErr := json.NewDecoder(r.Body).Decode(&logrec) + if decodeErr != nil { + logger.Debug(decodeErr) + return + } + + err := logrec.Validate() + if err != nil { + // log error + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", err.Error())) + // respond with error + w.WriteHeader(http.StatusBadRequest) + respondError(err, w, logger) + } else { + // now write to db + _, dbWriteErr := dbConn.WriteEventTelemetryV2(&logrec, logger) + if dbWriteErr != nil { + logger.Debug(dbWriteErr) + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", dbWriteErr)) + } else { + logrec.PrintRecordInfo(logger, OkMessage) + } + // respond with the created data + dumpEventAndRespond(logrec, w, logger) + } + + }).Methods("POST") + + // GET events/ + // get recorded telemetry record + router.HandleFunc("/api/v2/events/", func(w http.ResponseWriter, r *http.Request) { + // Parse query parameters + limit := 100 // Default limit + offset := 0 // Default offset + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } + } + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + // Parse search query if provided + var searchQuery map[string]interface{} + if searchStr := r.URL.Query().Get("q"); searchStr != "" { + if err := json.Unmarshal([]byte(searchStr), &searchQuery); err != nil { + w.WriteHeader(http.StatusBadRequest) + respondError(fmt.Errorf("invalid search query: %v", err), w, logger) + return + } + } + + // Get records + var records []persistence.EventTelemetryRecordV2 + var err error + if searchQuery != nil { + records, err = dbConn.SearchEventTelemetryV2(searchQuery, limit, offset, logger) + } else { + records, err = dbConn.ReadEventTelemetryV2(limit, offset, logger) + } + + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + // Write response + w.Header().Set("Content-Type", "application/json") + jsonData, err := json.Marshal(records) + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + if _, err := w.Write(jsonData); err != nil { + logger.Debug(err) + } + }).Methods("GET") +} diff --git a/server/scripts.go b/server/scripts.go new file mode 100644 index 0000000..4b12370 --- /dev/null +++ b/server/scripts.go @@ -0,0 +1,215 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gorilla/mux" +) + +func dumpScriptAndRespond(logrec interface{}, w http.ResponseWriter, logger *cli.Logger) { + // dump the telemetry record json data if requested + jsonData, responseDataErr := json.Marshal(logrec) + if responseDataErr == nil { + jsonString := string(jsonData) + if logger.PrintTrace { + logger.Trace(jsonString) + } + + // write response + w.Header().Set("Content-Type", "application/json") + _, responseErr := w.Write([]byte(jsonString)) + if responseErr != nil { + logger.Debug(responseErr) + } + } else { + logger.Debug(responseDataErr) + } +} + +func RouteScripts(router *mux.Router, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // POST scripts/ + // create new script telemetry record + // https://stackoverflow.com/a/26212073 + router.HandleFunc("/api/v1/scripts/", func(w http.ResponseWriter, r *http.Request) { + // parse given json data into a new record + logrec := persistence.ScriptTelemetryRecordV1{} + decodeErr := json.NewDecoder(r.Body).Decode(&logrec) + if decodeErr != nil { + logger.Debug(decodeErr) + return + } + + err := logrec.Validate() + if err != nil { + // log error + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", err.Error())) + // respond with error + w.WriteHeader(http.StatusBadRequest) + respondError(err, w, logger) + } else { + // now write to db + _, dbWriteErr := dbConn.WriteScriptTelemetryV1(&logrec, logger) + if dbWriteErr != nil { + logger.Debug(dbWriteErr) + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", dbWriteErr)) + } else { + logrec.PrintRecordInfo(logger, OkMessage) + } + // respond with the created data + dumpScriptAndRespond(logrec, w, logger) + } + + }).Methods("POST") + + router.HandleFunc("/api/v2/scripts/", func(w http.ResponseWriter, r *http.Request) { + // parse given json data into a new record + logrec := persistence.ScriptTelemetryRecordV2{} + decodeErr := json.NewDecoder(r.Body).Decode(&logrec) + if decodeErr != nil { + logger.Debug(decodeErr) + return + } + + // validate + err := logrec.Validate() + if err != nil { + // log error + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", err.Error())) + // respond with error + w.WriteHeader(http.StatusBadRequest) + respondError(err, w, logger) + } else { + // now write to db + _, dbWriteErr := dbConn.WriteScriptTelemetryV2(&logrec, logger) + if dbWriteErr != nil { + logger.Debug(dbWriteErr) + logrec.PrintRecordInfo(logger, fmt.Sprintf("[ {r}%s{!} ]", dbWriteErr)) + } else { + logrec.PrintRecordInfo(logger, OkMessage) + } + // respond with the created data + dumpScriptAndRespond(logrec, w, logger) + } + + }).Methods("POST") + + // GET scripts/ + // get recorded telemetry record + router.HandleFunc("/api/v1/scripts/", func(w http.ResponseWriter, r *http.Request) { + // Parse query parameters + limit := 100 // Default limit + offset := 0 // Default offset + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } + } + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + // Parse search query if provided + var searchQuery map[string]interface{} + if searchStr := r.URL.Query().Get("q"); searchStr != "" { + if err := json.Unmarshal([]byte(searchStr), &searchQuery); err != nil { + w.WriteHeader(http.StatusBadRequest) + respondError(fmt.Errorf("invalid search query: %v", err), w, logger) + return + } + } + + // Get records + var records []persistence.ScriptTelemetryRecordV1 + var err error + if searchQuery != nil { + records, err = dbConn.SearchScriptTelemetryV1(searchQuery, limit, offset, logger) + } else { + records, err = dbConn.ReadScriptTelemetryV1(limit, offset, logger) + } + + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + // Write response + w.Header().Set("Content-Type", "application/json") + jsonData, err := json.Marshal(records) + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + if _, err := w.Write(jsonData); err != nil { + logger.Debug(err) + } + }).Methods("GET") + + router.HandleFunc("/api/v2/scripts/", func(w http.ResponseWriter, r *http.Request) { + // Parse query parameters + limit := 100 // Default limit + offset := 0 // Default offset + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + limit = l + } + } + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + // Parse search query if provided + var searchQuery map[string]interface{} + if searchStr := r.URL.Query().Get("q"); searchStr != "" { + if err := json.Unmarshal([]byte(searchStr), &searchQuery); err != nil { + w.WriteHeader(http.StatusBadRequest) + respondError(fmt.Errorf("invalid search query: %v", err), w, logger) + return + } + } + + // Get records + var records []persistence.ScriptTelemetryRecordV2 + var err error + if searchQuery != nil { + records, err = dbConn.SearchScriptTelemetryV2(searchQuery, limit, offset, logger) + } else { + records, err = dbConn.ReadScriptTelemetryV2(limit, offset, logger) + } + + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + // Write response + w.Header().Set("Content-Type", "application/json") + jsonData, err := json.Marshal(records) + if err != nil { + logger.Debug(err) + w.WriteHeader(http.StatusInternalServerError) + respondError(err, w, logger) + return + } + + if _, err := w.Write(jsonData); err != nil { + logger.Debug(err) + } + }).Methods("GET") +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..a743170 --- /dev/null +++ b/server/server.go @@ -0,0 +1,55 @@ +package server + +import ( + "fmt" + "net/http" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gofrs/uuid" + "github.com/gorilla/mux" +) + +var ServerId uuid.UUID + +func NewRouter(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) http.Handler { + router := mux.NewRouter().StrictSlash(true) + + if opts.ScriptsTable != "" { + RouteScripts(router, opts, dbConn, logger) + } + if opts.EventsTable != "" { + RouteEvents(router, opts, dbConn, logger) + } + RouteStatus(router, opts, dbConn, logger) + + return router +} + +func Start(opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + ServerId = uuid.Must(uuid.NewV4()) + + router := NewRouter(opts, dbConn, logger) + + logger.Print(fmt.Sprintf("Server listening on %d...", opts.Port)) + if opts.Https { + logger.Fatal( + http.ListenAndServeTLS( + fmt.Sprintf(":%d", opts.Port), + fmt.Sprintf("%s.crt", opts.ExeName), + fmt.Sprintf("%s.key", opts.ExeName), + router, + )) + } else { + logger.Fatal( + http.ListenAndServe( + fmt.Sprintf(":%d", opts.Port), + router, + )) + } +} + +func GetStatus() string { + return "pass" // "pass", "fail" or "warn" +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..dc13a4c --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,239 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + "strings" + "testing" +) + +type mockDB struct{ persistence.Connection } + +func setupTestRouter() http.Handler { + opts := &cli.Options{ + ScriptsTable: "scripts", + EventsTable: "events", + } + db := &mockDB{} + logger := &cli.Logger{} + return NewRouter(opts, db, logger) +} + +func (m *mockDB) WriteScriptTelemetryV1(logrec *persistence.ScriptTelemetryRecordV1, logger *cli.Logger) (*persistence.Result, error) { + return &persistence.Result{ResultCode: 0, Message: "ok"}, nil +} +func (m *mockDB) WriteScriptTelemetryV2(logrec *persistence.ScriptTelemetryRecordV2, logger *cli.Logger) (*persistence.Result, error) { + return &persistence.Result{ResultCode: 0, Message: "ok"}, nil +} +func (m *mockDB) WriteEventTelemetryV2(logrec *persistence.EventTelemetryRecordV2, logger *cli.Logger) (*persistence.Result, error) { + return &persistence.Result{ResultCode: 0, Message: "ok"}, nil +} +func (m *mockDB) ReadScriptTelemetryV1(limit int, offset int, logger *cli.Logger) ([]persistence.ScriptTelemetryRecordV1, error) { + return []persistence.ScriptTelemetryRecordV1{}, nil +} +func (m *mockDB) ReadScriptTelemetryV2(limit int, offset int, logger *cli.Logger) ([]persistence.ScriptTelemetryRecordV2, error) { + return []persistence.ScriptTelemetryRecordV2{}, nil +} +func (m *mockDB) ReadEventTelemetryV2(limit int, offset int, logger *cli.Logger) ([]persistence.EventTelemetryRecordV2, error) { + return []persistence.EventTelemetryRecordV2{}, nil +} +func (m *mockDB) GetType() persistence.DBBackend { + return "mock" +} +func (m *mockDB) GetStatus(logger *cli.Logger) persistence.ConnectionStatus { + return persistence.ConnectionStatus{ + Status: "pass", + Version: "test", + Output: "mock", + } +} + +// ------------------------- +// Status Endpoint +// ------------------------- + +func TestStatusEndpoint(t *testing.T) { + router := setupTestRouter() + + t.Run("GET /api/v1/status returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v1/status", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + if w.Body.Len() == 0 { + t.Errorf("expected non-empty body") + } + }) +} + +// ------------------------- +// Script V1 +// ------------------------- + +func TestScriptV1Endpoints(t *testing.T) { + router := setupTestRouter() + + payload := `{ + "date":"2024-03-30", + "time":"08:45:00", + "username":"user", + "revit":"2021", + "revitbuild":"20240330_1234(x64)", + "sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4", + "pyrevit":"4.8", + "debug":false, + "config":false, + "commandname":"cmd", + "commanduniquename":"cmd.unique", + "commandbundle":"bundle", + "commandextension":"ext", + "resultcode":0, + "commandresults":{}, + "scriptpath":"/path/to/script", + "trace":{ + "engine":{"version":"1.0","syspath":[]}, + "ipy":"", + "clr":"" + } + }` + + t.Run("POST /api/v1/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("POST", "/api/v1/scripts/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) + + t.Run("GET /api/v1/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v1/scripts/", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) +} + +// ------------------------- +// Script V2 +// ------------------------- + +func TestScriptV2Endpoints(t *testing.T) { + router := setupTestRouter() + + payload := `{ + "meta":{"schema":"2.0"}, + "timestamp":"2024-03-30T08:45:00Z", + "username":"user", + "host_user":"host", + "revit":"2021", + "revitbuild":"20240330_1234(x64)", + "sessionid":"b3b1a2e0-4b5c-4d2a-8c3e-2b1a2e04b5c4", + "pyrevit":"4.8", + "clone":"main", + "debug":false, + "config":false, + "from_gui":false, + "exec_id":"execid", + "exec_timestamp":"2024-03-30T08:45:00Z", + "commandname":"cmd", + "commanduniquename":"cmd.unique", + "commandbundle":"bundle", + "commandextension":"ext", + "docname":"doc", + "docpath":"/path/to/doc", + "resultcode":0, + "commandresults":{}, + "scriptpath":"/path/to/script", + "trace":{ + "engine":{ + "type":"ironpython", + "version":"1.0", + "syspath":[], + "configs":{} + }, + "message":"" + } + }` + + t.Run("POST /api/v2/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("POST", "/api/v2/scripts/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) + + t.Run("GET /api/v2/scripts/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v2/scripts/", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) +} + +// ------------------------- +// Event V2 +// ------------------------- + +func TestEventV2Endpoints(t *testing.T) { + router := setupTestRouter() + + payload := `{ + "meta":{"schema":"2.0"}, + "timestamp":"2024-03-30T08:45:00Z", + "handler_id":"handler", + "type":"eventtype", + "args":{}, + "username":"user", + "host_user":"host", + "revit":"2021", + "revitbuild":"20240330_1234(x64)", + "cancellable":false, + "cancelled":false, + "docid":1, + "doctype":"type", + "doctemplate":"template", + "docname":"doc", + "docpath":"/path/to/doc", + "projectnum":"123", + "projectname":"proj" + }` + + t.Run("POST /api/v2/events/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("POST", "/api/v2/events/", strings.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) + + t.Run("GET /api/v2/events/ returns 200", func(t *testing.T) { + req := httptest.NewRequest("GET", "/api/v2/events/", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Errorf("expected 200 OK, got %d", w.Code) + } + }) +} diff --git a/server/status.go b/server/status.go new file mode 100644 index 0000000..91498c9 --- /dev/null +++ b/server/status.go @@ -0,0 +1,61 @@ +// https://inadarei.github.io/rfc-healthcheck/ +package server + +import ( + "encoding/json" + "net/http" + + "pyrevittelemetryserver/cli" + "pyrevittelemetryserver/persistence" + + "github.com/gorilla/mux" +) + +type ServerStatus struct { + Status string `json:"status"` + Version string `json:"version"` + Output string `json:"output"` + ServiceId string `json:"serviceid"` + Checks map[string]persistence.ConnectionStatus `json:"checks"` +} + +func prepareAndReportStatus(w http.ResponseWriter, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // create status report data + jsonData, responseDataErr := json.Marshal( + ServerStatus{ + Status: GetStatus(), + Version: opts.Version, + ServiceId: ServerId.String(), + Checks: map[string]persistence.ConnectionStatus{ + string(dbConn.GetType()): dbConn.GetStatus(logger), + }, + }) + if responseDataErr == nil { + jsonString := string(jsonData) + if logger.PrintTrace { + logger.Trace(jsonString) + } + + // write response + w.Header().Set("Content-Type", "application/health+json") + _, responseErr := w.Write([]byte(jsonString)) + if responseErr != nil { + logger.Debug(responseErr) + } + } else { + logger.Debug(responseDataErr) + } + +} + +func RouteStatus(router *mux.Router, opts *cli.Options, dbConn persistence.Connection, logger *cli.Logger) { + // GET scripts/ + // get recorded telemetry record + router.HandleFunc("/api/v1/status", func(w http.ResponseWriter, r *http.Request) { + prepareAndReportStatus(w, opts, dbConn, logger) + }).Methods("GET") + + router.HandleFunc("/api/v2/status", func(w http.ResponseWriter, r *http.Request) { + prepareAndReportStatus(w, opts, dbConn, logger) + }).Methods("GET") +}