[WIP] Improvements to support scale environment 2

.claude/agents/bitcoin-expert.md

@@ -1,3 +1,7 @@
+---
+name: bitcoin-expert
+---
+
 # bitcoin-expert
 
 You are a Bitcoin expert with deep technical knowledge of the original Bitcoin protocol as defined in Satoshi Nakamoto's whitepaper and implemented in Bitcoin SV (BSV). Your expertise encompasses cryptography, distributed systems, and the economic incentives that make Bitcoin work.

cmd/filereader/file_reader.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/bsv-blockchain/go-bt/v2"
@@ -37,6 +38,7 @@ import (
 	"github.com/bsv-blockchain/teranode/services/utxopersister"
 	"github.com/bsv-blockchain/teranode/settings"
 	"github.com/bsv-blockchain/teranode/stores/blob"
+	"github.com/bsv-blockchain/teranode/stores/blob/options"
 	"github.com/bsv-blockchain/teranode/ulogger"
 )
 
@@ -654,7 +656,18 @@ func getBlockStore(logger ulogger.Logger, settings *settings.Settings) blob.Stor
 		panic("blockstore config not found")
 	}
 
-	blockStore, err := blob.NewStore(logger, blockStoreURL)
+	var err error
+
+	hashPrefix := -2
+
+	if blockStoreURL.Query().Get("hashPrefix") != "" {
+		hashPrefix, err = strconv.Atoi(blockStoreURL.Query().Get("hashPrefix"))
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	blockStore, err := blob.NewStore(logger, blockStoreURL, options.WithHashPrefix(hashPrefix))
 	if err != nil {
 		panic(err)
 	}

cmd/seeder/seeder.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -42,6 +43,7 @@ import (
 	"github.com/bsv-blockchain/teranode/services/utxopersister"
 	"github.com/bsv-blockchain/teranode/settings"
 	"github.com/bsv-blockchain/teranode/stores/blob"
+	"github.com/bsv-blockchain/teranode/stores/blob/options"
 	bloboptions "github.com/bsv-blockchain/teranode/stores/blob/options"
 	"github.com/bsv-blockchain/teranode/stores/blockchain"
 	blockchainoptions "github.com/bsv-blockchain/teranode/stores/blockchain/options"
@@ -85,7 +87,7 @@ func usage(msg string) {
 //
 //nolint:gocognit // Requires refactoring to reduce cognitive complexity
 func Seeder(logger ulogger.Logger, appSettings *settings.Settings, inputDir string, hash string,
-	skipHeaders bool, skipUTXOs bool) {
+	skipHeaders bool, skipUTXOs bool, skipCheck bool) {
 	profilerAddr := appSettings.ProfilerAddr
 	if profilerAddr != "" {
 		go func() {
@@ -159,6 +161,7 @@ func Seeder(logger ulogger.Logger, appSettings *settings.Settings, inputDir stri
 			defer wg.Done()
 
 			logger.Infof("Processing headers...")
+			logger.Infof("Blockchain store: %s", appSettings.BlockChain.StoreURL)
 
 			// Process the headers
 			if err := processHeaders(ctx, logger, appSettings, headerFile); err != nil {
@@ -177,9 +180,10 @@ func Seeder(logger ulogger.Logger, appSettings *settings.Settings, inputDir stri
 			defer wg.Done()
 
 			logger.Infof("Processing UTXOs...")
+			logger.Infof("UTXO store: %s", appSettings.UtxoStore.UtxoStore.String())
 
 			// Process the UTXOs
-			if err := processUTXOs(ctx, logger, appSettings, utxoFile); err != nil {
+			if err := processUTXOs(ctx, logger, appSettings, utxoFile, skipCheck); err != nil {
 				logger.Errorf("Failed to process UTXOs: %v", err)
 				return
 			}
@@ -316,29 +320,42 @@ func processHeaders(ctx context.Context, logger ulogger.Logger, appSettings *set
 // processUTXOs reads the UTXO set from a file and stores it in the UTXO store.
 //
 //nolint:gocognit // Requires refactoring to reduce cognitive complexity
-func processUTXOs(ctx context.Context, logger ulogger.Logger, appSettings *settings.Settings, utxoFile string) error {
+func processUTXOs(ctx context.Context, logger ulogger.Logger, appSettings *settings.Settings, utxoFile string, skipCheck bool) error {
 	blockStoreURL := appSettings.Block.BlockStore
 	if blockStoreURL == nil {
 		return errors.NewConfigurationError("blockstore URL not found in config")
 	}
 
-	logger.Infof("Using blockStore at %s", blockStoreURL)
+	var err error
 
-	blockStore, err := blob.NewStore(logger, blockStoreURL)
-	if err != nil {
-		return errors.NewStorageError("failed to create blockStore", err)
+	hashPrefix := -2
+
+	if blockStoreURL.Query().Get("hashPrefix") != "" {
+		hashPrefix, err = strconv.Atoi(blockStoreURL.Query().Get("hashPrefix"))
+		if err != nil {
+			panic(err)
+		}
 	}
 
-	var exists bool
+	logger.Infof("Using blockStore at %s with hashPrefix %d", blockStoreURL, hashPrefix)
 
-	exists, err = blockStore.Exists(ctx, nil, fileformat.FileTypeDat, bloboptions.WithFilename("lastProcessed"))
+	blockStore, err := blob.NewStore(logger, blockStoreURL, options.WithHashPrefix(hashPrefix))
 	if err != nil {
-		return errors.NewStorageError("failed to check if lastProcessed.dat exists", err)
+		return errors.NewStorageError("failed to create blockStore", err)
 	}
 
-	if exists {
-		logger.Errorf("lastProcessed.dat exists, skipping UTXOs")
-		return nil
+	if !skipCheck {
+		var exists bool
+
+		exists, err = blockStore.Exists(ctx, nil, fileformat.FileTypeDat, bloboptions.WithFilename("lastProcessed"), bloboptions.WithNoHashPrefix())
+		if err != nil {
+			return errors.NewStorageError("failed to check if lastProcessed.dat exists", err)
+		}
+
+		if exists {
+			logger.Errorf("lastProcessed.dat exists, skipping UTXOs")
+			return nil
+		}
 	}
 
 	logger.Infof("Using utxostore at %s", appSettings.UtxoStore.UtxoStore)
@@ -470,7 +487,7 @@ func processUTXOs(ctx context.Context, logger ulogger.Logger, appSettings *setti
 
 	heightStr := fmt.Sprintf("%d\n", height)
 
-	if err = blockStore.Set(ctx, nil, fileformat.FileTypeDat, []byte(heightStr), bloboptions.WithFilename("lastProcessed")); err != nil {
+	if err = blockStore.Set(ctx, nil, fileformat.FileTypeDat, []byte(heightStr), bloboptions.WithFilename("lastProcessed"), bloboptions.WithNoHashPrefix()); err != nil {
 		return errors.NewStorageError("failed to write height of %d to lastProcessed.dat", height, err)
 	}
 

cmd/settings/Settings.go

@@ -16,12 +16,140 @@ package settings
 
 import (
 	"encoding/json"
+	"fmt"
+	"reflect"
+	"strings"
 
 	"github.com/bsv-blockchain/teranode/settings"
 	"github.com/bsv-blockchain/teranode/ulogger"
 	"github.com/ordishs/gocore"
 )
 
+// marshalSortedJSON marshals a struct to JSON with sorted keys at all levels.
+// It recursively sorts struct fields alphabetically but leaves arrays/slices unchanged.
+func marshalSortedJSON(v interface{}, indent string) ([]byte, error) {
+	sorted := sortValue(v)
+	return json.MarshalIndent(sorted, "", indent)
+}
+
+// sortValue recursively sorts struct fields but leaves slices/arrays unchanged
+func sortValue(v interface{}) interface{} {
+	if v == nil {
+		return nil
+	}
+
+	val := reflect.ValueOf(v)
+	typ := reflect.TypeOf(v)
+
+	// Check if the type implements json.Marshaler - if so, preserve it as-is
+	// to maintain custom JSON marshaling behavior (like hash types that output as hex strings)
+	if _, ok := v.(json.Marshaler); ok {
+		return v
+	}
+
+	// Also check if pointer to this type implements json.Marshaler
+	if val.CanAddr() {
+		if _, ok := val.Addr().Interface().(json.Marshaler); ok {
+			return v
+		}
+	}
+
+	// Additional check: if the pointer type implements json.Marshaler, create a pointer
+	// This matches Go's JSON marshaling behavior which automatically takes addresses when needed
+	ptrType := reflect.PtrTo(typ)
+	if ptrType.Implements(reflect.TypeOf((*json.Marshaler)(nil)).Elem()) {
+		// Create a new pointer to this value to enable custom marshaling
+		newVal := reflect.New(typ)
+		newVal.Elem().Set(val)
+		return newVal.Interface()
+	}
+
+	// Check if this type has a method set that implements json.Marshaler
+	// This handles cases where the interface check above might fail
+	if typ.Implements(reflect.TypeOf((*json.Marshaler)(nil)).Elem()) {
+		return v
+	}
+
+	// Handle pointers
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		return sortValue(val.Elem().Interface())
+	}
+
+	// Handle arrays - preserve byte arrays as-is for potential hex encoding
+	if val.Kind() == reflect.Array {
+		// Special case: byte arrays (like [32]byte hashes) should be preserved as-is
+		if val.Type().Elem().Kind() == reflect.Uint8 {
+			return v
+		}
+
+		// For other arrays, recursively process elements but don't sort the array itself
+		result := make([]interface{}, val.Len())
+		for i := 0; i < val.Len(); i++ {
+			result[i] = sortValue(val.Index(i).Interface())
+		}
+		return result
+	}
+
+	// Handle slices - leave them unchanged, only process nested structs
+	if val.Kind() == reflect.Slice {
+		if val.Len() == 0 {
+			return v
+		}
+
+		// Special case: convert byte slices to hex strings instead of base64
+		if val.Type().Elem().Kind() == reflect.Uint8 {
+			bytes := make([]byte, val.Len())
+			for i := 0; i < val.Len(); i++ {
+				bytes[i] = uint8(val.Index(i).Uint())
+			}
+			return fmt.Sprintf("%x", bytes)
+		}
+
+		// For other slices, recursively process elements but don't sort the slice itself
+		result := make([]interface{}, val.Len())
+		for i := 0; i < val.Len(); i++ {
+			result[i] = sortValue(val.Index(i).Interface())
+		}
+		return result
+	}
+
+	// Handle structs
+	if val.Kind() == reflect.Struct {
+		result := make(map[string]interface{})
+
+		for i := 0; i < val.NumField(); i++ {
+			field := typ.Field(i)
+			fieldVal := val.Field(i)
+
+			// Skip unexported fields
+			if !fieldVal.CanInterface() {
+				continue
+			}
+
+			// Get JSON tag name, fallback to field name
+			jsonTag := field.Tag.Get("json")
+			fieldName := field.Name
+			if jsonTag != "" && jsonTag != "-" {
+				// Handle json:",omitempty" and similar
+				if idx := strings.Index(jsonTag, ","); idx != -1 {
+					fieldName = jsonTag[:idx]
+				} else {
+					fieldName = jsonTag
+				}
+			}
+
+			result[fieldName] = sortValue(fieldVal.Interface())
+		}
+		return result
+	}
+
+	// For all other types, return as-is
+	return v
+}
+
 // PrintSettings prints the application settings, version, and commit information in a structured format.
 //
 // This function is used to display the current application configuration and runtime details
@@ -41,7 +169,7 @@ func PrintSettings(logger ulogger.Logger, settings *settings.Settings, version,
 	stats := gocore.Config().Stats()
 	logger.Infof("STATS\n%s\nVERSION\n-------\n%s (%s)\n\n", stats, version, commit)
 
-	settingsJSON, err := json.MarshalIndent(settings, "", "  ")
+	settingsJSON, err := marshalSortedJSON(settings, "  ")
 	if err != nil {
 		logger.Errorf("Failed to marshal settings: %v", err)
 	} else {