Skip to content

Commit 30458c3

Browse files
Merge pull request #307 from ava-labs/add-general-indexer
Add general indexer
2 parents d716346 + 91513d7 commit 30458c3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+3576
-351
lines changed

api/server/server.go

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ var (
3232
errUnknownLockOption = errors.New("invalid lock options")
3333
)
3434

35+
type RouteAdder interface {
36+
AddRoute(handler *common.HTTPHandler, lock *sync.RWMutex, base, endpoint string, loggingWriter io.Writer) error
37+
}
38+
3539
// Server maintains the HTTP router
3640
type Server struct {
3741
// log this server writes to
@@ -96,25 +100,34 @@ func (s *Server) DispatchTLS(certFile, keyFile string) error {
96100
return http.ServeTLS(listener, s.handler, certFile, keyFile)
97101
}
98102

99-
// RegisterChain registers the API endpoints associated with this chain That is,
100-
// add <route, handler> pairs to server so that http calls can be made to the vm
101-
func (s *Server) RegisterChain(chainName string, ctx *snow.Context, vmIntf interface{}) {
102-
vm, ok := vmIntf.(common.VM)
103-
if !ok {
104-
return
105-
}
103+
// RegisterChain registers the API endpoints associated with this chain. That is,
104+
// add <route, handler> pairs to server so that API calls can be made to the VM.
105+
// This method runs in a goroutine to avoid a deadlock in the event that the caller
106+
// holds the engine's context lock. Namely, this could happen when the P-Chain is
107+
// creating a new chain and holds the P-Chain's lock when this function is held,
108+
// and at the same time the server's lock is held due to an API call and is trying
109+
// to grab the P-Chain's lock.
110+
func (s *Server) RegisterChain(chainName string, ctx *snow.Context, engine common.Engine) {
111+
go s.registerChain(chainName, ctx, engine)
112+
}
113+
114+
func (s *Server) registerChain(chainName string, ctx *snow.Context, engine common.Engine) {
115+
var (
116+
handlers map[string]*common.HTTPHandler
117+
err error
118+
)
106119

107120
ctx.Lock.Lock()
108-
handlers, err := vm.CreateHandlers()
121+
handlers, err = engine.GetVM().CreateHandlers()
109122
ctx.Lock.Unlock()
110123
if err != nil {
111-
s.log.Error("Failed to create %s handlers: %s", chainName, err)
124+
s.log.Error("failed to create %s handlers: %s", chainName, err)
112125
return
113126
}
114127

115128
httpLogger, err := s.factory.MakeChain(chainName, "http")
116129
if err != nil {
117-
s.log.Error("Failed to create new http logger: %s", err)
130+
s.log.Error("failed to create new http logger: %s", err)
118131
return
119132
}
120133

@@ -123,15 +136,15 @@ func (s *Server) RegisterChain(chainName string, ctx *snow.Context, vmIntf inter
123136
defaultEndpoint := "bc/" + ctx.ChainID.String()
124137

125138
// Register each endpoint
126-
for extension, service := range handlers {
139+
for extension, handler := range handlers {
127140
// Validate that the route being added is valid
128141
// e.g. "/foo" and "" are ok but "\n" is not
129142
_, err := url.ParseRequestURI(extension)
130143
if extension != "" && err != nil {
131144
s.log.Error("could not add route to chain's API handler because route is malformed: %s", err)
132145
continue
133146
}
134-
if err := s.AddChainRoute(service, ctx, defaultEndpoint, extension, httpLogger); err != nil {
147+
if err := s.AddChainRoute(handler, ctx, defaultEndpoint, extension, httpLogger); err != nil {
135148
s.log.Error("error adding route: %s", err)
136149
}
137150
}

chains/manager.go

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ type Manager interface {
6565
ForceCreateChain(ChainParameters)
6666

6767
// Add a registrant [r]. Every time a chain is
68-
// created, [r].RegisterChain([new chain]) is called
68+
// created, [r].RegisterChain([new chain]) is called.
6969
AddRegistrant(Registrant)
7070

7171
// Given an alias, return the ID of the chain associated with that alias
@@ -149,7 +149,8 @@ type manager struct {
149149
ids.Aliaser
150150
ManagerConfig
151151

152-
registrants []Registrant // Those notified when a chain is created
152+
// Those notified when a chain is created
153+
registrants []Registrant
153154

154155
unblocked bool
155156
blockedChains []ChainParameters
@@ -240,7 +241,20 @@ func (m *manager) ForceCreateChain(chainParams ChainParameters) {
240241
m.Log.AssertNoError(m.Alias(chainParams.ID, chainParams.ID.String()))
241242

242243
// Notify those that registered to be notified when a new chain is created
243-
m.notifyRegistrants(chain.Name, chain.Ctx, chain.VM)
244+
m.notifyRegistrants(chain.Name, chain.Ctx, chain.Engine)
245+
246+
// Tell the chain to start processing messages.
247+
// If the X or P Chain panics, do not attempt to recover
248+
if m.CriticalChains.Contains(chainParams.ID) {
249+
go chain.Ctx.Log.RecoverAndPanic(chain.Handler.Dispatch)
250+
} else {
251+
go chain.Ctx.Log.RecoverAndExit(chain.Handler.Dispatch, func() {
252+
chain.Ctx.Log.Error("Chain with ID: %s was shutdown due to a panic", chainParams.ID)
253+
})
254+
}
255+
256+
// Allows messages to be routed to the new chain
257+
m.ManagerConfig.Router.AddChain(chain.Handler)
244258
}
245259

246260
// Create a chain
@@ -382,17 +396,6 @@ func (m *manager) buildChain(chainParams ChainParameters, sb Subnet) (*chain, er
382396
return nil, err
383397
}
384398

385-
// Allows messages to be routed to the new chain
386-
m.ManagerConfig.Router.AddChain(chain.Handler)
387-
388-
// If the X or P Chain panics, do not attempt to recover
389-
if m.CriticalChains.Contains(chainParams.ID) {
390-
go ctx.Log.RecoverAndPanic(chain.Handler.Dispatch)
391-
} else {
392-
go ctx.Log.RecoverAndExit(chain.Handler.Dispatch, func() {
393-
ctx.Log.Error("Chain with ID: %s was shutdown due to a panic", chainParams.ID)
394-
})
395-
}
396399
return chain, nil
397400
}
398401

@@ -696,9 +699,9 @@ func (m *manager) LookupVM(alias string) (ids.ID, error) { return m.VMManager.Lo
696699

697700
// Notify registrants [those who want to know about the creation of chains]
698701
// that the specified chain has been created
699-
func (m *manager) notifyRegistrants(name string, ctx *snow.Context, vm interface{}) {
702+
func (m *manager) notifyRegistrants(name string, ctx *snow.Context, engine common.Engine) {
700703
for _, registrant := range m.registrants {
701-
go registrant.RegisterChain(name, ctx, vm)
704+
registrant.RegisterChain(name, ctx, engine)
702705
}
703706
}
704707

chains/registrant.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,13 @@ package chains
55

66
import (
77
"github.com/ava-labs/avalanchego/snow"
8+
"github.com/ava-labs/avalanchego/snow/engine/common"
89
)
910

1011
// Registrant can register the existence of a chain
1112
type Registrant interface {
12-
RegisterChain(name string, ctx *snow.Context, vm interface{})
13+
// Called when the chain described by [ctx] and [engine] is created
14+
// This function is called before the chain starts processing messages
15+
// [engine] should be an avalanche.Engine or snowman.Engine
16+
RegisterChain(name string, ctx *snow.Context, engine common.Engine)
1317
}

go.sum

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
3333
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
3434
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
3535
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
36-
github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
3736
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
3837
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
3938
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -86,7 +85,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2
8685
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
8786
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
8887
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
89-
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
9088
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
9189
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
9290
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

indexer/client.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package indexer
2+
3+
import (
4+
"time"
5+
6+
"github.com/ava-labs/avalanchego/utils/rpc"
7+
)
8+
9+
type Client struct {
10+
rpc.EndpointRequester
11+
}
12+
13+
// NewClient creates a client that can interact with an index via HTTP API calls.
14+
// [host] is the host to make API calls to (e.g. http://1.2.3.4:9650).
15+
// [endpoint] is the path to the index endpoint (e.g. /ext/index/C/block or /ext/index/X/tx).
16+
func NewClient(host, endpoint string, requestTimeout time.Duration) *Client {
17+
return &Client{
18+
EndpointRequester: rpc.NewEndpointRequester(host, endpoint, "index", requestTimeout),
19+
}
20+
}
21+
22+
func (c *Client) GetContainerRange(args *GetContainerRangeArgs) ([]FormattedContainer, error) {
23+
var response GetContainerRangeResponse
24+
err := c.SendRequest("getContainerRange", args, &response)
25+
return response.Containers, err
26+
}
27+
28+
func (c *Client) GetContainerByIndex(args *GetContainer) (FormattedContainer, error) {
29+
var response FormattedContainer
30+
err := c.SendRequest("getContainerByIndex", args, &response)
31+
return response, err
32+
}
33+
34+
func (c *Client) GetLastAccepted(args *GetLastAcceptedArgs) (FormattedContainer, error) {
35+
var response FormattedContainer
36+
err := c.SendRequest("getLastAccepted", args, &response)
37+
return response, err
38+
}
39+
40+
func (c *Client) GetIndex(args *GetIndexArgs) (GetIndexResponse, error) {
41+
var response GetIndexResponse
42+
err := c.SendRequest("getIndex", args, &response)
43+
return response, err
44+
}
45+
46+
func (c *Client) IsAccepted(args *GetIndexArgs) (bool, error) {
47+
var response bool
48+
err := c.SendRequest("isAccepted", args, &response)
49+
return response, err
50+
}

indexer/client_test.go

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package indexer
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/ava-labs/avalanchego/ids"
8+
"github.com/ava-labs/avalanchego/utils/formatting"
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
type mockClient struct {
13+
f func(reply interface{}) error
14+
}
15+
16+
func (mc *mockClient) SendRequest(_ string, _ interface{}, reply interface{}) error {
17+
return mc.f(reply)
18+
}
19+
20+
func TestIndexClient(t *testing.T) {
21+
assert := assert.New(t)
22+
client := NewClient("http://localhost:9650", "/ext/index/C/block", time.Minute)
23+
24+
// Test GetIndex
25+
client.EndpointRequester = &mockClient{
26+
f: func(reply interface{}) error {
27+
*(reply.(*GetIndexResponse)) = GetIndexResponse{Index: 5}
28+
return nil
29+
},
30+
}
31+
index, err := client.GetIndex(&GetIndexArgs{ContainerID: ids.Empty, Encoding: formatting.Hex})
32+
assert.NoError(err)
33+
assert.EqualValues(5, index.Index)
34+
35+
// Test GetLastAccepted
36+
id := ids.GenerateTestID()
37+
client.EndpointRequester = &mockClient{
38+
f: func(reply interface{}) error {
39+
*(reply.(*FormattedContainer)) = FormattedContainer{ID: id}
40+
return nil
41+
},
42+
}
43+
container, err := client.GetLastAccepted(&GetLastAcceptedArgs{Encoding: formatting.Hex})
44+
assert.NoError(err)
45+
assert.EqualValues(id, container.ID)
46+
47+
// Test GetContainerRange
48+
id = ids.GenerateTestID()
49+
client.EndpointRequester = &mockClient{
50+
f: func(reply interface{}) error {
51+
*(reply.(**GetContainerRangeResponse)) = &GetContainerRangeResponse{Containers: []FormattedContainer{{ID: id}}}
52+
return nil
53+
},
54+
}
55+
containers, err := client.GetContainerRange(&GetContainerRangeArgs{StartIndex: 1, NumToFetch: 10, Encoding: formatting.Hex})
56+
assert.NoError(err)
57+
assert.Len(containers, 1)
58+
assert.EqualValues(id, containers[0].ID)
59+
60+
// Test IsAccepted
61+
client.EndpointRequester = &mockClient{
62+
f: func(reply interface{}) error {
63+
*(reply.(*bool)) = true
64+
return nil
65+
},
66+
}
67+
isAccepted, err := client.IsAccepted(&GetIndexArgs{ContainerID: ids.Empty, Encoding: formatting.Hex})
68+
assert.NoError(err)
69+
assert.True(isAccepted)
70+
}

indexer/container.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package indexer
2+
3+
import "github.com/ava-labs/avalanchego/ids"
4+
5+
// Container is something that gets accepted
6+
// (a block, transaction or vertex)
7+
type Container struct {
8+
// ID of this container
9+
ID ids.ID `serialize:"true"`
10+
// Byte representation of this container
11+
Bytes []byte `serialize:"true"`
12+
// Unix time, in nanoseconds, at which this container was accepted by this node
13+
Timestamp int64 `serialize:"true"`
14+
}

0 commit comments

Comments
 (0)