Merge pull request #8 from dipdup-io/GO-63-Add-Source-Config

Change Hasura configuration structure:
* Now `source` is a structure but not string
* Remove flag `add_source`

Author: aopoltorzhicky

.gitignore

@@ -5,6 +5,8 @@
 *.so
 *.dylib
 .env
+.idea
+.DS_Store
 
 # Test binary, built with `go test -c`
 *.test

README.md

@@ -233,4 +233,8 @@ head, err := tzkt.GetHead(ctx)
 if err != nil {
 	log.Panic(err)
 }
-```
+```
+
+### `hasura`
+
+Go wrapper for Hasura metadata methods, read docs [here](hasura/README.md)

config/config.go

@@ -52,18 +52,28 @@ type Database struct {
 
 // Hasura -
 type Hasura struct {
-	URL                string `yaml:"url" validate:"required,url"`
-	Secret             string `yaml:"admin_secret" validate:"required"`
-	Source             string `yaml:"source" validate:"omitempty"`
-	RowsLimit          uint64 `yaml:"select_limit" validate:"gt=0"`
-	EnableAggregations bool   `yaml:"allow_aggregation"`
-	AddSource          bool   `yaml:"add_source"`
-	Rest               *bool  `yaml:"rest"`
+	URL                string        `yaml:"url" validate:"required,url"`
+	Secret             string        `yaml:"admin_secret" validate:"required"`
+	RowsLimit          uint64        `yaml:"select_limit" validate:"gt=0"`
+	EnableAggregations bool          `yaml:"allow_aggregation"`
+	Source             *HasuraSource `yaml:"source"`
+	Rest               *bool         `yaml:"rest"`
+}
+
+type HasuraSource struct {
+	Name                  string `yaml:"name" validate:"required"`
+	DatabaseHost          string `yaml:"database_host"`
+	UsePreparedStatements bool   `yaml:"use_prepared_statements"`
+	IsolationLevel        string `yaml:"isolation_level"`
 }
 
 // UnmarshalYAML -
 func (h *Hasura) UnmarshalYAML(unmarshal func(interface{}) error) error {
-	h.Source = "default"
+	h.Source = &HasuraSource{
+		Name:                  "default",
+		UsePreparedStatements: false,
+		IsolationLevel:        "read-committed",
+	}
 
 	type plain Hasura
 	return unmarshal((*plain)(h))

hasura/README.md

@@ -0,0 +1,19 @@
+# Go wrapper for Hasura metadata methods
+
+Golang's library for registration data source in Hasura service
+
+## Configuration
+
+```yaml
+hasura:
+  url: string             # hasura url, required
+  admin_secret: string    # required
+  select_limit: int       
+  allow_aggregation: bool
+  source:
+    name: string                  # name of data source, required. For more info, [hasura docs](https://hasura.io/docs/latest/api-reference/metadata-api/source/#metadata-pg-add-source-syntax).
+    database_host: string         # host of datasource, if omitted, used host from database config
+    use_prepared_statements: bool # if set to true the server prepares statement before executing on the source database (default: false)
+    isolation_level: bool         # The transaction isolation level in which the queries made to the source will be run with (options: read-committed | repeatable-read | serializable) (default: read-committed)
+  rest: bool              # should REST endpoints be created?
+```

hasura/api.go

@@ -66,7 +66,7 @@ func (api *API) get(ctx context.Context, endpoint string, args map[string]string
 	return api.client.Do(req)
 }
 
-//nolint
+// nolint
 func (api *API) post(ctx context.Context, endpoint string, args map[string]string, body interface{}, output interface{}) error {
 	url, err := api.buildURL(endpoint, args)
 	if err != nil {
@@ -124,15 +124,27 @@ func (api *API) Health(ctx context.Context) error {
 
 // AddSource -
 func (api *API) AddSource(ctx context.Context, hasura *config.Hasura, cfg config.Database) error {
+	host := cfg.Host
+	if hasura.Source.DatabaseHost != "" {
+		host = hasura.Source.DatabaseHost
+	}
+
+	databaseUrl := DatabaseUrl(fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", cfg.User, cfg.Password, host, cfg.Port, cfg.Database))
+
+	isolationLevel := "read-committed"
+	if hasura.Source.IsolationLevel != "" {
+		isolationLevel = hasura.Source.IsolationLevel
+	}
+
 	req := Request{
 		Type: "pg_add_source",
 		Args: map[string]interface{}{
-			"name": hasura.Source,
+			"name": hasura.Source.Name,
 			"configuration": Configuration{
 				ConnectionInfo: ConnectionInfo{
-					DatabaseUrl:           DatabaseUrl(fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database)),
-					UsePreparedStatements: true,
-					IsolationLevel:        "read-committed",
+					DatabaseUrl:           databaseUrl,
+					UsePreparedStatements: hasura.Source.UsePreparedStatements,
+					IsolationLevel:        isolationLevel,
 				},
 			},
 			"replace_configuration": true,

hasura/hasura.go

@@ -64,7 +64,7 @@ func Create(ctx context.Context, args GenerateArgs) error {
 
 	checkHealth(ctx, api)
 
-	if args.Config.AddSource {
+	if args.Config.Source != nil {
 		log.Info().Msg("Adding source...")
 		if err := api.AddSource(ctx, args.Config, args.DatabaseConfig); err != nil {
 			return err
@@ -85,13 +85,13 @@ func Create(ctx context.Context, args GenerateArgs) error {
 	// Find our source in the existing metadata
 	var selectedSource *Source = nil
 	for idx := range export.Sources {
-		if export.Sources[idx].Name == args.Config.Source {
+		if export.Sources[idx].Name == args.Config.Source.Name {
 			selectedSource = &export.Sources[idx]
 			break
 		}
 	}
 	if selectedSource == nil {
-		return errors.Errorf("Source '%s' not found on exported metadata", args.Config.Source)
+		return errors.Errorf("Source '%s' not found on exported metadata", args.Config.Source.Name)
 	}
 
 	log.Info().Msg("Merging metadata...")
@@ -123,15 +123,15 @@ func Create(ctx context.Context, args GenerateArgs) error {
 
 	log.Info().Msg("Tracking views...")
 	for i := range args.Views {
-		if err := api.TrackTable(ctx, args.Views[i], args.Config.Source); err != nil {
+		if err := api.TrackTable(ctx, args.Views[i], args.Config.Source.Name); err != nil {
 			if !strings.Contains(err.Error(), "view/table already tracked") {
 				return err
 			}
 		}
-		if err := api.DropSelectPermissions(ctx, args.Views[i], args.Config.Source, "user"); err != nil {
+		if err := api.DropSelectPermissions(ctx, args.Views[i], args.Config.Source.Name, "user"); err != nil {
 			log.Warn().Err(err).Msg("")
 		}
-		if err := api.CreateSelectPermissions(ctx, args.Views[i], args.Config.Source, "user", Permission{
+		if err := api.CreateSelectPermissions(ctx, args.Views[i], args.Config.Source.Name, "user", Permission{
 			Limit:     args.Config.RowsLimit,
 			AllowAggs: args.Config.EnableAggregations,
 			Columns:   Columns{"*"},
@@ -155,7 +155,7 @@ func Create(ctx context.Context, args GenerateArgs) error {
 func Generate(hasura config.Hasura, cfg config.Database, models ...interface{}) (*Metadata, error) {
 	schema := getSchema(cfg)
 	source := Source{
-		Name:   hasura.Source,
+		Name:   hasura.Source.Name,
 		Tables: make([]Table, 0),
 	}
 	for _, model := range models {

hasura/hasura_test.go

@@ -142,7 +142,11 @@ func TestGenerate(t *testing.T) {
 				hasura: config.Hasura{
 					EnableAggregations: true,
 					RowsLimit:          5,
-					Source:             "mysql",
+					Source: &config.HasuraSource{
+						Name:                  "mysql",
+						UsePreparedStatements: true,
+						IsolationLevel:        "read-committed",
+					},
 				},
 				models: []interface{}{
 					&testTable{}, &testTable2{}, &testTable3{}, &testTable4{},