Merge pull request #162 from abicky/integrate-with-opentelemetry

Integrate with OpenTelemetry

Author: siburu

.github/workflows/test.yml

@@ -56,6 +56,13 @@ jobs:
     needs:
       - relayer-build
       - tendermint-build
+    strategy:
+      matrix:
+        enable-telemetry:
+          - 'false'
+          - 'true'
+    env:
+      YRLY_ENABLE_TELEMETRY: ${{ matrix.enable-telemetry }}
     steps:
       - uses: actions/checkout@v4
       - name: Restore relayer binary cache
@@ -87,6 +94,13 @@ jobs:
     needs:
       - relayer-build
       - tendermint-build
+    strategy:
+      matrix:
+        enable-telemetry:
+          - 'false'
+          - 'true'
+    env:
+      YRLY_ENABLE_TELEMETRY: ${{ matrix.enable-telemetry }}
     steps:
       - uses: actions/checkout@v4
       - name: Restore relayer binary cache

README.md

@@ -103,3 +103,108 @@ You can use it by specifying the package name of the proto definition correspond
   }
 }
 ```
+
+## OpenTelemetry integration
+
+OpenTelemetry integration can be enabled by specifying the `--enable-telemetry` flag or by setting `YRLY_ENABLE_TELEMETRY` environment variable to true.
+To see an example setup, refer to [examples/opentelemetry-integration](examples/opentelemetry-integration).
+
+### Configurations
+
+You can configure its behavior using environment variables supported by the Go SDK, as listed in the [Compliance of Implementations with Specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/spec-compliance-matrix.md#environment-variables).
+
+In addition to these environment variables, yui-relayer supports the following variables, which are not available in the Go SDK:
+
+* OTEL_PROPAGATORS
+* OTEL_TRACES_EXPORTER
+    - Note that `"zipkin"` is not supported
+* OTEL_METRICS_EXPORTER
+* OTEL_LOGS_EXPORTER
+* OTEL_EXPORTER_PROMETHEUS_HOST
+* OTEL_EXPORTER_PROMETHEUS_PORT
+* OTEL_EXPORTER_CONSOLE_TRACES_WRITER
+* OTEL_EXPORTER_CONSOLE_LOGS_WRITER
+* OTEL_EXPORTER_CONSOLE_METRICS_WRITER
+
+The `OTEL_EXPORTER_CONSOLE_*_WRITER` variables are specific to yui-relayer and allow you to change the output destination of the standard output exporters. To redirect output to standard error, set the value to `stderr`.
+
+For more information about OpenTelemetry environment variables, refer to the [OpenTelemetry Environment Variable Specification](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables).
+
+
+When OpenTelemetry integration is enabled, the OTLP log exporter is enabled by default and you may want to disable ordinal logs.
+In this case, you can disable them by setting `.global.logger.output` to `"null"` in the yui-relayer configuration file.
+
+### Add spans and span attributes in external modules
+
+#### Using tracing bridges
+
+The Relayer provides OpenTelemetry tracing bridges: `otelcore.Chain` and `otelcore.Prover`.
+These bridges add tracing to the primary methods defined in the Chain and Prover interfaces.
+You can use the tracing bridges by returning them in `ChainConfig.Build` and `ProverConfig.Build`:
+
+```go
+var tracer = otel.Tracer("example.com/my-module")
+
+func (c ChainConfig) Build() (core.Chain, error) {
+	chain := buildChainFromConfig(c)
+	return otelcore.NewChain(chain, tracer), nil
+}
+
+func (c ProverConfig) Build(chain core.Chain) (core.Prover, error) {
+	prover := buildProverFromConfig(c)
+	return otelcore.NewProver(prover, chain.ChainID(), tracer), nil
+}
+```
+
+If you need to access the original Chain and Prover implementations, you can use `coreutil.UnwrapChain` and `coreutil.UnwrapProver`:
+
+```go
+// The case where a ProvableChain contains a chain struct (module.Chain)
+chain, err := coreutil.UnwrapChain[module.Chain](provableChain)
+
+// The case where a ProvableChain contains a chain struct pointer (*module.Chain)
+chainPtr, err := coreutil.UnwrapChain[*module.Chain](provableChain)
+```
+
+Note that, if you call methods defined in your Chain module and Prover module directly, tracing data will not be recorded.
+
+#### Manual tracing
+
+In addition to using the tracing bridges, you can manually create spans when needed:
+
+```go
+var tracer = otel.Tracer("example.com/my-module")
+
+func someFunction(ctx context.Context) {
+	ctx, span := tracer.Start(ctx, "someFunction")
+	defer span.End()
+
+	// -- snip --
+}
+```
+
+If a function or method receives a `core.QueryContext`, you can use `core.StartTraceWithQueryContext` to create a span:
+
+```go
+func (c *Chain) QuerySomething(ctx core.QueryContext) (any, error) {
+	ctx, span := core.StartTraceWithQueryContext(tracer, ctx, "Chain.QuerySomething", core.WithChainAttributes(c.ChainID()))
+	defer span.End()
+
+	// -- snip --
+```
+
+You can also add span attributes as follows:
+
+```go
+func (c *Chain) GetMsgResult(ctx context.Context, id core.MsgID) (core.MsgResult, error) {
+	msgID, ok := id.(*MsgID)
+	if !ok {
+		return nil, fmt.Errorf("unexpected message id type: %T", id)
+	}
+
+	span := trace.SpanFromContext(ctx)
+	span.SetAttributes(semconv.TxHashKey.String(msgID.TxHash))
+
+	// -- snip --
+}
+```

chains/tendermint/chain.go

@@ -12,6 +12,10 @@ import (
 
 	"cosmossdk.io/errors"
 	"github.com/avast/retry-go"
+	"github.com/hyperledger-labs/yui-relayer/otelcore/semconv"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	rpcclient "github.com/cometbft/cometbft/rpc/client"
 	rpchttp "github.com/cometbft/cometbft/rpc/client/http"
@@ -182,6 +186,7 @@ func (c *Chain) sendMsgs(ctx context.Context, msgs []sdk.Msg) (*sdk.TxResponse,
 		// CheckTx failed
 		return nil, fmt.Errorf("CheckTx failed: %v", errors.ABCIError(res.Codespace, res.Code, res.RawLog))
 	}
+	trace.SpanFromContext(ctx).SetAttributes(semconv.TxHashKey.String(res.TxHash))
 
 	// wait for tx being committed
 	if resTx, err := c.waitForCommit(ctx, res.TxHash); err != nil {
@@ -203,6 +208,9 @@ func (c *Chain) sendMsgs(ctx context.Context, msgs []sdk.Msg) (*sdk.TxResponse,
 }
 
 func (c *Chain) rawSendMsgs(ctx context.Context, msgs []sdk.Msg) (*sdk.TxResponse, bool, error) {
+	ctx, span := tracer.Start(ctx, "Chain.rawSendMsgs", core.WithChainAttributes(c.ChainID()))
+	defer span.End()
+
 	// Instantiate the client context
 	// NOTE: Although cosmos-sdk does not currently use CmdContext in Context.QueryWithData,
 	//   set ctx to clientCtx in case cosmos-sdk uses it in the future.
@@ -212,6 +220,7 @@ func (c *Chain) rawSendMsgs(ctx context.Context, msgs []sdk.Msg) (*sdk.TxRespons
 	// Query account details
 	txf, err := prepareFactory(clientCtx, c.TxFactory(0))
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return nil, false, err
 	}
 
@@ -220,6 +229,7 @@ func (c *Chain) rawSendMsgs(ctx context.Context, msgs []sdk.Msg) (*sdk.TxRespons
 	// If users pass gas adjustment, then calculate gas
 	_, adjusted, err := CalculateGas(clientCtx.QueryWithData, txf, msgs...)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return nil, false, err
 	}
 
@@ -229,35 +239,41 @@ func (c *Chain) rawSendMsgs(ctx context.Context, msgs []sdk.Msg) (*sdk.TxRespons
 	// Build the transaction builder
 	txb, err := txf.BuildUnsignedTx(msgs...)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return nil, false, err
 	}
 
 	// Attach the signature to the transaction
 	err = tx.Sign(ctx, txf, c.config.Key, txb, false)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return nil, false, err
 	}
 
 	// Generate the transaction bytes
 	txBytes, err := clientCtx.TxConfig.TxEncoder()(txb.GetTx())
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return nil, false, err
 	}
 
 	// Broadcast those bytes
 	res, err := clientCtx.BroadcastTx(txBytes)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return nil, false, err
 	}
 
 	// transaction was executed, log the success or failure using the tx response code
 	// NOTE: error is nil, logic should use the returned error to determine if the
 	// transaction was successfully executed.
 	if res.Code != 0 {
+		span.SetStatus(codes.Error, "non-zero response code")
 		c.LogFailedTx(res, err, msgs)
 		return res, false, nil
 	}
 
+	span.SetAttributes(semconv.TxHashKey.String(res.TxHash))
 	c.LogSuccessTx(res, msgs)
 	return res, true, nil
 }
@@ -429,6 +445,9 @@ func (c *Chain) GetMsgResult(ctx context.Context, id core.MsgID) (core.MsgResult
 		return nil, fmt.Errorf("unexpected message id type: %T", id)
 	}
 
+	span := trace.SpanFromContext(ctx)
+	span.SetAttributes(semconv.TxHashKey.String(msgID.TxHash))
+
 	// find tx
 	resTx, err := c.waitForCommit(ctx, msgID.TxHash)
 	if err != nil {
@@ -558,6 +577,10 @@ func newRPCClient(addr string, timeout time.Duration) (*rpchttp.HTTP, error) {
 		return nil, err
 	}
 
+	// NOTE: The Cosmos SDK typically does not propagate parent contexts when making JSON-RPC calls,
+	//   so most spans recorded by otelhttp do not have a parent span.
+	//   cf. https://github.com/cosmos/cosmos-sdk/issues/24500
+	httpClient.Transport = otelhttp.NewTransport(httpClient.Transport)
 	httpClient.Timeout = timeout
 	rpcClient, err := rpchttp.NewWithClient(addr, "/websocket", httpClient)
 	if err != nil {

chains/tendermint/cmd/keys.go

@@ -7,6 +7,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/crypto/hd"
 	"github.com/hyperledger-labs/yui-relayer/chains/tendermint"
 	"github.com/hyperledger-labs/yui-relayer/config"
+	"github.com/hyperledger-labs/yui-relayer/coreutil"
 	"github.com/spf13/cobra"
 )
 
@@ -40,7 +41,11 @@ func keysAddCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			chain := c.Chain.(*tendermint.Chain)
+
+			chain, err := coreutil.UnwrapChain[*tendermint.Chain](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Chain", args[0])
+			}
 
 			var keyName string
 			if len(args) == 2 {
@@ -100,7 +105,11 @@ func keysRestoreCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			chain := c.Chain.(*tendermint.Chain)
+
+			chain, err := coreutil.UnwrapChain[*tendermint.Chain](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Chain", args[0])
+			}
 
 			if chain.KeyExists(keyName) {
 				return errKeyExists(keyName)
@@ -136,7 +145,11 @@ func keysShowCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			chain := c.Chain.(*tendermint.Chain)
+
+			chain, err := coreutil.UnwrapChain[*tendermint.Chain](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Chain", args[0])
+			}
 
 			var keyName string
 			if len(args) == 2 {
@@ -177,7 +190,11 @@ func keysListCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			chain := c.Chain.(*tendermint.Chain)
+
+			chain, err := coreutil.UnwrapChain[*tendermint.Chain](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Chain", args[0])
+			}
 
 			info, err := chain.Keybase.List()
 			if err != nil {

chains/tendermint/cmd/light.go

@@ -8,6 +8,7 @@ import (
 	tmclient "github.com/cosmos/ibc-go/v8/modules/light-clients/07-tendermint"
 	"github.com/hyperledger-labs/yui-relayer/chains/tendermint"
 	"github.com/hyperledger-labs/yui-relayer/config"
+	"github.com/hyperledger-labs/yui-relayer/coreutil"
 	"github.com/spf13/cobra"
 )
 
@@ -40,8 +41,15 @@ func initLightCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			chain := c.Chain.(*tendermint.Chain)
-			prover := c.Prover.(*tendermint.Prover)
+
+			chain, err := coreutil.UnwrapChain[*tendermint.Chain](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Chain", args[0])
+			}
+			prover, err := coreutil.UnwrapProver[*tendermint.Prover](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Prover", args[0])
+			}
 
 			db, df, err := prover.NewLightDB(cmd.Context())
 			if err != nil {
@@ -96,7 +104,11 @@ func updateLightCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			prover := c.Prover.(*tendermint.Prover)
+
+			prover, err := coreutil.UnwrapProver[*tendermint.Prover](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Prover", args[0])
+			}
 
 			bh, err := prover.GetLatestLightHeader(cmd.Context())
 			if err != nil {
@@ -129,8 +141,15 @@ func lightHeaderCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			chain := c.Chain.(*tendermint.Chain)
-			prover := c.Prover.(*tendermint.Prover)
+
+			chain, err := coreutil.UnwrapChain[*tendermint.Chain](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Chain", args[0])
+			}
+			prover, err := coreutil.UnwrapProver[*tendermint.Prover](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Prover", args[0])
+			}
 
 			var header *tmclient.Header
 
@@ -188,7 +207,11 @@ func deleteLightCmd(ctx *config.Context) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			prover := c.Prover.(*tendermint.Prover)
+
+			prover, err := coreutil.UnwrapProver[*tendermint.Prover](c)
+			if err != nil {
+				return fmt.Errorf("Chain %q is not a tendermint.Prover", args[0])
+			}
 
 			err = prover.DeleteLightDB()
 			if err != nil {